fix(Core/Misc): few improvements to ut8 handling (#2455)

src/common/Containers.h

@@ -8,6 +8,10 @@
 #define ACORE_CONTAINERS_H
 
 #include "Define.h"
+#include <algorithm>
+#include <exception>
+#include <iterator>
+#include <utility>
 #include <list>
 
 //! Because circular includes are bad
@@ -15,6 +19,46 @@ extern uint32 urand(uint32 min, uint32 max);
 
 namespace acore
 {
+    template<class T>
+    constexpr inline T* AddressOrSelf(T* ptr)
+    {
+        return ptr;
+    }
+
+    template<class T>
+    constexpr inline T* AddressOrSelf(T& not_ptr)
+    {
+        return std::addressof(not_ptr);
+    }
+
+    template <class T>
+    class CheckedBufferOutputIterator
+    {
+        public:
+            using iterator_category = std::output_iterator_tag;
+            using value_type = void;
+            using pointer = T*;
+            using reference = T&;
+            using difference_type = std::ptrdiff_t;
+
+            CheckedBufferOutputIterator(T* buf, size_t n) : _buf(buf), _end(buf+n) {}
+
+            T& operator*() const { check(); return *_buf; }
+            CheckedBufferOutputIterator& operator++() { check(); ++_buf; return *this; }
+            CheckedBufferOutputIterator operator++(int) { CheckedBufferOutputIterator v = *this; operator++(); return v; }
+
+            size_t remaining() const { return (_end - _buf); }
+
+        private:
+            T* _buf;
+            T* _end;
+            void check() const
+            {
+                if (!(_buf < _end))
+                    throw std::out_of_range("index");
+            }
+    };
+
     namespace Containers
     {
         template<class T>

src/common/Utilities/Util.cpp

@@ -16,6 +16,8 @@
 #include "Errors.h" // for ASSERT
 #include <ace/TSS_T.h>
 #include <array>
+#include <cwchar>
+#include <string>
 
 typedef ACE_TSS<SFMTRand> SFMTRandTSS;
 static SFMTRandTSS sfmtRand;
@@ -251,22 +253,29 @@ bool IsIPAddrInNetwork(ACE_INET_Addr const& net, ACE_INET_Addr const& addr, ACE_
 }
 
 /// create PID file
-uint32 CreatePIDFile(const std::string& filename)
+uint32 CreatePIDFile(std::string const& filename)
 {
     FILE* pid_file = fopen(filename.c_str(), "w");
-    if (pid_file == NULL)
+    if (pid_file == nullptr)
         return 0;
 
+    uint32 pid = GetPID();
+
+    fprintf(pid_file, "%u", pid);
+    fclose(pid_file);
+
+    return pid;
+}
+
+uint32 GetPID()
+{
 #ifdef _WIN32
     DWORD pid = GetCurrentProcessId();
 #else
     pid_t pid = getpid();
 #endif
 
-    fprintf(pid_file, "%u", pid );
+    return uint32(pid);
-    fclose(pid_file);
-
-    return (uint32)pid;
 }
 
 size_t utf8length(std::string& utf8str)
@@ -275,9 +284,9 @@ size_t utf8length(std::string& utf8str)
     {
         return utf8::distance(utf8str.c_str(), utf8str.c_str()+utf8str.size());
     }
-    catch(std::exception)
+    catch(std::exception const&)
     {
-        utf8str = "";
+        utf8str.clear();
         return 0;
     }
 }
@@ -297,9 +306,9 @@ void utf8truncate(std::string& utf8str, size_t len)
         char* oend = utf8::utf16to8(wstr.c_str(), wstr.c_str()+wstr.size(), &utf8str[0]);
         utf8str.resize(oend-(&utf8str[0]));                 // remove unused tail
     }
-    catch(std::exception)
+    catch(std::exception const&)
     {
-        utf8str = "";
+        utf8str.clear();
     }
 }
 
@@ -307,24 +316,30 @@ bool Utf8toWStr(char const* utf8str, size_t csize, wchar_t* wstr, size_t& wsize)
 {
     try
     {
-        size_t len = utf8::distance(utf8str, utf8str+csize);
+        acore::CheckedBufferOutputIterator<wchar_t> out(wstr, wsize);
-        if (len > wsize)
+        out = utf8::utf8to16(utf8str, utf8str+csize, out);
+        wsize -= out.remaining(); // remaining unused space
+        wstr[wsize] = L'\0';
+    }
+    catch(std::exception const&)
+    {
+        // Replace the converted string with an error message if there is enough space
+        // Otherwise just return an empty string
+        const wchar_t* errorMessage = L"An error occurred converting string from UTF-8 to WStr";
+        std::size_t errorMessageLength = std::char_traits<wchar_t>::length(errorMessage);
+        if (wsize >= errorMessageLength)
+        {
+            std::wcscpy(wstr, errorMessage);
+            wsize = std::char_traits<wchar_t>::length(wstr);
+        }
+        else if (wsize > 0)
         {
-            if (wsize > 0)
             wstr[0] = L'\0';
             wsize = 0;
-            return false;
         }
+        else
+            wsize = 0;
 
-        wsize = len;
-        utf8::utf8to16(utf8str, utf8str+csize, wstr);
-        wstr[len] = L'\0';
-    }
-    catch(std::exception)
-    {
-        if (wsize > 0)
-            wstr[0] = L'\0';
-        wsize = 0;
         return false;
     }
 
@@ -361,16 +376,16 @@ bool WStrToUtf8(wchar_t* wstr, size_t size, std::string& utf8str)
         }
         utf8str = utf8str2;
     }
-    catch(std::exception)
+    catch(std::exception const&)
     {
-        utf8str = "";
+        utf8str.clear();
         return false;
     }
 
     return true;
 }
 
-bool WStrToUtf8(std::wstring wstr, std::string& utf8str)
+bool WStrToUtf8(std::wstring const& wstr, std::string& utf8str)
 {
     try
     {
@@ -384,9 +399,9 @@ bool WStrToUtf8(std::wstring wstr, std::string& utf8str)
         }
         utf8str = utf8str2;
     }
-    catch(std::exception)
+    catch(std::exception const&)
     {
-        utf8str = "";
+        utf8str.clear();
         return false;
     }
 
@@ -395,14 +410,23 @@ bool WStrToUtf8(std::wstring wstr, std::string& utf8str)
 
 typedef wchar_t const* const* wstrlist;
 
-std::wstring GetMainPartOfName(std::wstring wname, uint32 declension)
+void wstrToUpper(std::wstring& str)
+{
+    std::transform(str.begin(), str.end(), str.begin(), wcharToUpper);
+}
+
+void wstrToLower(std::wstring& str)
+{
+    std::transform(str.begin(), str.end(), str.begin(), wcharToLower);
+}
+
+std::wstring GetMainPartOfName(std::wstring const& wname, uint32 declension)
 {
     // supported only Cyrillic cases
     if (wname.empty() || !isCyrillicCharacter(wname[0]) || declension > 5)
         return wname;
 
     // Important: end length must be <= MAX_INTERNAL_PLAYER_NAME-MAX_PLAYER_NAME (3 currently)
-
     static std::wstring const a_End    = { wchar_t(0x0430), wchar_t(0x0000) };
     static std::wstring const o_End    = { wchar_t(0x043E), wchar_t(0x0000) };
     static std::wstring const ya_End   = { wchar_t(0x044F), wchar_t(0x0000) };
@@ -467,7 +491,7 @@ bool consoleToUtf8(const std::string& conStr, std::string& utf8str)
 #if AC_PLATFORM == AC_PLATFORM_WINDOWS
     std::wstring wstr;
     wstr.resize(conStr.size());
-    OemToCharBuffW(&conStr[0], &wstr[0], conStr.size());
+    OemToCharBuffW(&conStr[0], &wstr[0], uint32(conStr.size()));
 
     return WStrToUtf8(wstr, utf8str);
 #else
@@ -477,7 +501,7 @@ bool consoleToUtf8(const std::string& conStr, std::string& utf8str)
 #endif
 }
 
-bool Utf8FitTo(const std::string& str, std::wstring search)
+bool Utf8FitTo(const std::string& str, std::wstring const& search)
 {
     std::wstring temp;
 
@@ -515,13 +539,24 @@ void vutf8printf(FILE* out, const char *str, va_list* ap)
     size_t wtemp_len = 32*1024-1;
     Utf8toWStr(temp_buf, temp_len, wtemp_buf, wtemp_len);
 
-    CharToOemBuffW(&wtemp_buf[0], &temp_buf[0], wtemp_len+1);
+    CharToOemBuffW(&wtemp_buf[0], &temp_buf[0], uint32(wtemp_len + 1));
     fprintf(out, "%s", temp_buf);
 #else
     vfprintf(out, str, *ap);
 #endif
 }
 
+bool Utf8ToUpperOnlyLatin(std::string& utf8String)
+{
+    std::wstring wstr;
+    if (!Utf8toWStr(utf8String, wstr))
+        return false;
+
+    std::transform(wstr.begin(), wstr.end(), wstr.begin(), wcharToUpperOnlyLatin);
+
+    return WStrToUtf8(wstr, utf8String);
+}
+
 std::string ByteArrayToHexStr(uint8 const* bytes, uint32 arrayLen, bool reverse /* = false */)
 {
     int32 init = 0;
@@ -545,3 +580,41 @@ std::string ByteArrayToHexStr(uint8 const* bytes, uint32 arrayLen, bool reverse
 
     return ss.str();
 }
+
+void HexStrToByteArray(std::string const& str, uint8* out, bool reverse /*= false*/)
+{
+    // string must have even number of characters
+    if (str.length() & 1)
+        return;
+
+    int32 init = 0;
+    int32 end = int32(str.length());
+    int8 op = 1;
+
+    if (reverse)
+    {
+        init = int32(str.length() - 2);
+        end = -2;
+        op = -1;
+    }
+
+    uint32 j = 0;
+    for (int32 i = init; i != end; i += 2 * op)
+    {
+        char buffer[3] = { str[i], str[i + 1], '\0' };
+        out[j++] = uint8(strtoul(buffer, nullptr, 16));
+    }
+}
+
+bool StringToBool(std::string const& str)
+{
+    std::string lowerStr = str;
+    std::transform(str.begin(), str.end(), lowerStr.begin(), ::tolower);
+    return lowerStr == "1" || lowerStr == "true" || lowerStr == "yes";
+}
+
+bool StringContainsStringI(std::string const& haystack, std::string const& needle)
+{
+    return haystack.end() !=
+        std::search(haystack.begin(), haystack.end(), needle.begin(), needle.end(), [](char c1, char c2) { return std::toupper(c1) == std::toupper(c2); });
+}

src/common/Utilities/Util.h

@@ -9,8 +9,9 @@
 
 #include "Define.h"
 #include "Errors.h"
-
+#include "Containers.h"
 #include <algorithm>
+#include <cctype>
 #include <string>
 #include <vector>
 #include <list>
@@ -127,18 +128,21 @@ inline T RoundToInterval(T& num, T floor, T ceil)
 
 // UTF8 handling
 bool Utf8toWStr(const std::string& utf8str, std::wstring& wstr);
+
 // in wsize==max size of buffer, out wsize==real string size
 bool Utf8toWStr(char const* utf8str, size_t csize, wchar_t* wstr, size_t& wsize);
+
 inline bool Utf8toWStr(const std::string& utf8str, wchar_t* wstr, size_t& wsize)
 {
     return Utf8toWStr(utf8str.c_str(), utf8str.size(), wstr, wsize);
 }
 
-bool WStrToUtf8(std::wstring wstr, std::string& utf8str);
+bool WStrToUtf8(std::wstring const& wstr, std::string& utf8str);
 // size==real string size
 bool WStrToUtf8(wchar_t* wstr, size_t size, std::string& utf8str);
 
-size_t utf8length(std::string& utf8str);                    // set string to "" if invalid utf8 sequence
+// set string to "" if invalid utf8 sequence
+size_t utf8length(std::string& utf8str);
 void utf8truncate(std::string& utf8str, size_t len);
 
 inline bool isBasicLatinCharacter(wchar_t wchar)
@@ -308,23 +312,17 @@ inline wchar_t wcharToLower(wchar_t wchar)
     return wchar;
 }
 
-inline void wstrToUpper(std::wstring& str)
+void wstrToUpper(std::wstring& str);
-{
+void wstrToLower(std::wstring& str);
-    std::transform( str.begin(), str.end(), str.begin(), wcharToUpper );
-}
 
-inline void wstrToLower(std::wstring& str)
+std::wstring GetMainPartOfName(std::wstring const& wname, uint32 declension);
-{
-    std::transform( str.begin(), str.end(), str.begin(), wcharToLower );
-}
-
-std::wstring GetMainPartOfName(std::wstring wname, uint32 declension);
 
 bool utf8ToConsole(const std::string& utf8str, std::string& conStr);
 bool consoleToUtf8(const std::string& conStr, std::string& utf8str);
-bool Utf8FitTo(const std::string& str, std::wstring search);
+bool Utf8FitTo(const std::string& str, std::wstring const& search);
 void utf8printf(FILE* out, const char *str, ...);
 void vutf8printf(FILE* out, const char *str, va_list* ap);
+bool Utf8ToUpperOnlyLatin(std::string& utf8String);
 
 bool IsIPAddress(char const* ipaddress);
 
@@ -335,8 +333,18 @@ bool IsIPAddrInNetwork(ACE_INET_Addr const& net, ACE_INET_Addr const& addr, ACE_
 std::string GetAddressString(ACE_INET_Addr const& addr);
 
 uint32 CreatePIDFile(const std::string& filename);
+uint32 GetPID();
 
 std::string ByteArrayToHexStr(uint8 const* bytes, uint32 length, bool reverse = false);
+void HexStrToByteArray(std::string const& str, uint8* out, bool reverse = false);
+bool StringToBool(std::string const& str);
+
+bool StringContainsStringI(std::string const& haystack, std::string const& needle);
+template <typename T>
+inline bool ValueContainsStringI(std::pair<T, std::string> const& haystack, std::string const& needle)
+{
+    return StringContainsStringI(haystack.second, needle);
+}
 #endif
 
 //handler for operations on large flags

src/server/game/Accounts/AccountMgr.cpp

@@ -24,8 +24,8 @@ namespace AccountMgr
         if (utf8length(password) > MAX_PASS_STR)
             return AOR_PASS_TOO_LONG;                           // password's too long
 
-        normalizeString(username);
+        Utf8ToUpperOnlyLatin(username);
-        normalizeString(password);
+        Utf8ToUpperOnlyLatin(password);
 
         if (GetId(username))
             return AOR_NAME_ALREDY_EXIST;                       // username does already exist
@@ -134,8 +134,8 @@ namespace AccountMgr
         if (utf8length(newPassword) > MAX_PASS_STR)
             return AOR_PASS_TOO_LONG;                           // password's too long
 
-        normalizeString(newUsername);
+        Utf8ToUpperOnlyLatin(newUsername);
-        normalizeString(newPassword);
+        Utf8ToUpperOnlyLatin(newPassword);
 
         stmt = LoginDatabase.GetPreparedStatement(LOGIN_UPD_USERNAME);
 
@@ -164,8 +164,8 @@ namespace AccountMgr
             return AOR_PASS_TOO_LONG;                           // password's too long
         }
 
-        normalizeString(username);
+        Utf8ToUpperOnlyLatin(username);
-        normalizeString(newPassword);
+        Utf8ToUpperOnlyLatin(newPassword);
 
         PreparedStatement* stmt = LoginDatabase.GetPreparedStatement(LOGIN_UPD_PASSWORD);
 
@@ -228,8 +228,8 @@ namespace AccountMgr
         if (!GetName(accountId, username))
             return false;
 
-        normalizeString(username);
+        Utf8ToUpperOnlyLatin(username);
-        normalizeString(password);
+        Utf8ToUpperOnlyLatin(password);
 
         PreparedStatement* stmt = LoginDatabase.GetPreparedStatement(LOGIN_SEL_CHECK_PASSWORD);
         stmt->setUInt32(0, accountId);
@@ -249,24 +249,6 @@ namespace AccountMgr
         return (result) ? (*result)[0].GetUInt64() : 0;
     }
 
-    bool normalizeString(std::string& utf8String)
-    {
-        wchar_t buffer[MAX_ACCOUNT_STR + 1];
-
-        size_t maxLength = MAX_ACCOUNT_STR;
-        if (!Utf8toWStr(utf8String, buffer, maxLength))
-            return false;
-#ifdef _MSC_VER
-#pragma warning(disable: 4996)
-#endif
-        std::transform(&buffer[0], buffer + maxLength, &buffer[0], wcharToUpperOnlyLatin);
-#ifdef _MSC_VER
-#pragma warning(default: 4996)
-#endif
-
-        return WStrToUtf8(buffer, maxLength, utf8String);
-    }
-
     std::string CalculateShaPassHash(std::string const& name, std::string const& password)
     {
         SHA1Hash sha;

src/server/game/Accounts/AccountMgr.h

@@ -38,7 +38,6 @@ namespace AccountMgr
     uint32 GetCharactersCount(uint32 accountId);
     std::string CalculateShaPassHash(std::string const& name, std::string const& password);
 
-    bool normalizeString(std::string& utf8String);
     bool IsPlayerAccount(uint32 gmlevel);
     bool IsGMAccount(uint32 gmlevel);
     bool IsAdminAccount(uint32 gmlevel);

src/server/scripts/Commands/cs_account.cpp

@@ -147,7 +147,7 @@ public:
             return false;
 
         std::string accountName = account;
-        if (!AccountMgr::normalizeString(accountName))
+        if (!Utf8ToUpperOnlyLatin(accountName))
         {
             handler->PSendSysMessage(LANG_ACCOUNT_NOT_EXIST, accountName.c_str());
             handler->SetSentErrorMessage(true);
@@ -419,7 +419,7 @@ public:
         {
             ///- Convert Account name to Upper Format
             accountName = account;
-            if (!AccountMgr::normalizeString(accountName))
+            if (!Utf8ToUpperOnlyLatin(accountName))
             {
                 handler->PSendSysMessage(LANG_ACCOUNT_NOT_EXIST, accountName.c_str());
                 handler->SetSentErrorMessage(true);
@@ -485,7 +485,7 @@ public:
         if (isAccountNameGiven)
         {
             targetAccountName = arg1;
-            if (!AccountMgr::normalizeString(targetAccountName))
+            if (!Utf8ToUpperOnlyLatin(targetAccountName))
             {
                 handler->PSendSysMessage(LANG_ACCOUNT_NOT_EXIST, targetAccountName.c_str());
                 handler->SetSentErrorMessage(true);
@@ -597,7 +597,7 @@ public:
             return false;
 
         std::string accountName = account;
-        if (!AccountMgr::normalizeString(accountName))
+        if (!Utf8ToUpperOnlyLatin(accountName))
         {
             handler->PSendSysMessage(LANG_ACCOUNT_NOT_EXIST, accountName.c_str());
             handler->SetSentErrorMessage(true);

src/server/scripts/Commands/cs_ban.cpp

@@ -166,7 +166,7 @@ public:
         switch (mode)
         {
             case BAN_ACCOUNT:
-                if (!AccountMgr::normalizeString(nameOrIP))
+                if (!Utf8ToUpperOnlyLatin(nameOrIP))
                 {
                     handler->PSendSysMessage(LANG_ACCOUNT_NOT_EXIST, nameOrIP.c_str());
                     handler->SetSentErrorMessage(true);
@@ -253,7 +253,7 @@ public:
             return false;
 
         std::string accountName = nameStr;
-        if (!AccountMgr::normalizeString(accountName))
+        if (!Utf8ToUpperOnlyLatin(accountName))
         {
             handler->PSendSysMessage(LANG_ACCOUNT_NOT_EXIST, accountName.c_str());
             handler->SetSentErrorMessage(true);
@@ -713,7 +713,7 @@ public:
         switch (mode)
         {
             case BAN_ACCOUNT:
-                if (!AccountMgr::normalizeString(nameOrIP))
+                if (!Utf8ToUpperOnlyLatin(nameOrIP))
                 {
                     handler->PSendSysMessage(LANG_ACCOUNT_NOT_EXIST, nameOrIP.c_str());
                     handler->SetSentErrorMessage(true);

src/server/scripts/Commands/cs_character.cpp

@@ -764,7 +764,7 @@ public:
             return false;
 
         std::string accountName = accountStr;
-        if (!AccountMgr::normalizeString(accountName))
+        if (!Utf8ToUpperOnlyLatin(accountName))
         {
             handler->PSendSysMessage(LANG_ACCOUNT_NOT_EXIST, accountName.c_str());
             handler->SetSentErrorMessage(true);

src/server/scripts/Commands/cs_lookup.cpp

@@ -1296,7 +1296,7 @@ public:
         char* limitStr = strtok(nullptr, " ");
         int32 limit = limitStr ? atoi(limitStr) : -1;
 
-        if (!AccountMgr::normalizeString
+        if (!Utf8ToUpperOnlyLatin
             (account))
             return false;
 

src/server/scripts/Commands/cs_misc.cpp

@@ -2330,7 +2330,7 @@ public:
             return false;
 
         std::string accountName = nameStr;
-        if (!AccountMgr::normalizeString(accountName))
+        if (!Utf8ToUpperOnlyLatin(accountName))
         {
             handler->PSendSysMessage(LANG_ACCOUNT_NOT_EXIST, accountName.c_str());
             handler->SetSentErrorMessage(true);

src/server/worldserver/RemoteAccess/RASocket.cpp

@@ -173,7 +173,7 @@ int RASocket::check_access_level(const std::string& user)
 {
     std::string safeUser = user;
 
-    AccountMgr::normalizeString(safeUser);
+    Utf8ToUpperOnlyLatin(safeUser);
 
     PreparedStatement* stmt = LoginDatabase.GetPreparedStatement(LOGIN_SEL_ACCOUNT_ACCESS);
     stmt->setString(0, safeUser);
@@ -204,10 +204,10 @@ int RASocket::check_access_level(const std::string& user)
 int RASocket::check_password(const std::string& user, const std::string& pass)
 {
     std::string safe_user = user;
-    AccountMgr::normalizeString(safe_user);
+    Utf8ToUpperOnlyLatin(safe_user);
 
     std::string safe_pass = pass;
-    AccountMgr::normalizeString(safe_pass);
+    Utf8ToUpperOnlyLatin(safe_pass);
 
     std::string hash = AccountMgr::CalculateShaPassHash(safe_user, safe_pass);