боты русифицированы
This commit is contained in:
Binary file not shown.
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""Build the static mod-playerbots ruRU SQL update from a reviewed TSV file."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import re
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
INPUT = ROOT / "playerbots-phrases-ru-reviewed.tsv"
|
||||
OUTPUT = (
|
||||
ROOT
|
||||
/ "modules/mod-playerbots/data/sql/playerbots/updates/"
|
||||
/ "2026_07_30_00_ai_playerbot_russian_texts.sql"
|
||||
)
|
||||
EXPECTED_FIELDS = ("id", "key", "original_en", "translation_ru")
|
||||
PLACEHOLDER_RE = re.compile(r"%[A-Za-z_][A-Za-z0-9_]*|<[^>]+>")
|
||||
|
||||
|
||||
def sql_quote(value: str) -> str:
|
||||
return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'"
|
||||
|
||||
|
||||
def load_rows() -> list[dict[str, str]]:
|
||||
with INPUT.open(encoding="utf-8-sig", newline="") as input_file:
|
||||
reader = csv.DictReader(input_file, delimiter="\t")
|
||||
if tuple(reader.fieldnames or ()) != EXPECTED_FIELDS:
|
||||
raise RuntimeError(
|
||||
f"Unexpected TSV columns: {reader.fieldnames}; "
|
||||
f"expected {EXPECTED_FIELDS}"
|
||||
)
|
||||
rows = list(reader)
|
||||
|
||||
if not rows:
|
||||
raise RuntimeError(f"No translations found in {INPUT}")
|
||||
|
||||
seen_ids: set[int] = set()
|
||||
for line_number, row in enumerate(rows, start=2):
|
||||
try:
|
||||
row_id = int(row["id"])
|
||||
except ValueError as error:
|
||||
raise RuntimeError(
|
||||
f"{INPUT}:{line_number}: invalid id {row['id']!r}"
|
||||
) from error
|
||||
|
||||
if row_id in seen_ids:
|
||||
raise RuntimeError(f"{INPUT}:{line_number}: duplicate id {row_id}")
|
||||
seen_ids.add(row_id)
|
||||
|
||||
if not row["key"]:
|
||||
raise RuntimeError(f"{INPUT}:{line_number}: empty key")
|
||||
if not row["translation_ru"].strip():
|
||||
raise RuntimeError(f"{INPUT}:{line_number}: empty translation")
|
||||
|
||||
source_placeholders = Counter(PLACEHOLDER_RE.findall(row["original_en"]))
|
||||
target_placeholders = Counter(PLACEHOLDER_RE.findall(row["translation_ru"]))
|
||||
if source_placeholders != target_placeholders:
|
||||
raise RuntimeError(
|
||||
f"{INPUT}:{line_number}: placeholder mismatch: "
|
||||
f"{source_placeholders} != {target_placeholders}"
|
||||
)
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def write_sql(rows: list[dict[str, str]]) -> None:
|
||||
lines = [
|
||||
"-- MoonWell static Russian localization for mod-playerbots.",
|
||||
f"-- Generated from {INPUT.name}; edit the TSV and rebuild, not this file.",
|
||||
"-- Placeholder names and counts are validated; their order may differ in Russian.",
|
||||
"SET NAMES utf8mb4;",
|
||||
"",
|
||||
"START TRANSACTION;",
|
||||
]
|
||||
for row in rows:
|
||||
lines.append(
|
||||
"UPDATE `ai_playerbot_texts` "
|
||||
f"SET `text_loc8`={sql_quote(row['translation_ru'])} "
|
||||
f"WHERE `id`={int(row['id'])} AND `name`={sql_quote(row['key'])};"
|
||||
)
|
||||
lines.extend(["COMMIT;", ""])
|
||||
OUTPUT.write_text("\n".join(lines), encoding="utf-8")
|
||||
print(f"Wrote {len(rows)} reviewed translations to {OUTPUT}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
write_sql(load_rows())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""Export all playerbot chat phrases for manual Russian translation review."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUTPUT = ROOT / "doc" / "playerbots-phrases-ru-review.tsv"
|
||||
|
||||
|
||||
def load_rows() -> list[tuple[str, str, str, str]]:
|
||||
sql = (
|
||||
"SELECT id,HEX(name),HEX(text),HEX(COALESCE(text_loc8,'')) "
|
||||
"FROM ai_playerbot_texts ORDER BY id"
|
||||
)
|
||||
command = [
|
||||
"docker",
|
||||
"compose",
|
||||
"exec",
|
||||
"-T",
|
||||
"ac-database",
|
||||
"bash",
|
||||
"-lc",
|
||||
f'mysql -N -s -uroot -p"$MYSQL_ROOT_PASSWORD" '
|
||||
f'acore_playerbots -e "{sql}" 2>/dev/null',
|
||||
]
|
||||
output = subprocess.check_output(command, cwd=ROOT, text=True)
|
||||
rows: list[tuple[str, str, str, str]] = []
|
||||
|
||||
for line in output.splitlines():
|
||||
parts = line.split("\t")
|
||||
if len(parts) != 4:
|
||||
continue
|
||||
row_id, name_hex, source_hex, russian_hex = parts
|
||||
rows.append(
|
||||
(
|
||||
row_id,
|
||||
bytes.fromhex(name_hex).decode("utf-8"),
|
||||
bytes.fromhex(source_hex).decode("utf-8", errors="replace"),
|
||||
bytes.fromhex(russian_hex).decode("utf-8", errors="replace"),
|
||||
)
|
||||
)
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rows = load_rows()
|
||||
OUTPUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
with OUTPUT.open("w", encoding="utf-8", newline="") as output:
|
||||
writer = csv.writer(output, delimiter="\t", quoting=csv.QUOTE_MINIMAL)
|
||||
writer.writerow(("id", "key", "original_en", "translation_ru"))
|
||||
writer.writerows(rows)
|
||||
|
||||
print(f"Wrote {len(rows)} phrases to {OUTPUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,314 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""Generate deterministic Cyrillic playerbot character and guild name pools."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUTPUT = (
|
||||
ROOT
|
||||
/ "modules/mod-playerbots/data/sql/characters/updates/"
|
||||
/ "2026_07_30_00_playerbots_russian_names.sql"
|
||||
)
|
||||
NAMES_PER_CATEGORY = 250
|
||||
UPSTREAM_NAMES = (
|
||||
ROOT
|
||||
/ "modules/mod-playerbots/data/sql/characters/base/playerbots_names.sql"
|
||||
)
|
||||
|
||||
|
||||
CATEGORIES = {
|
||||
0: (
|
||||
["Ал", "Бор", "Вел", "Влад", "Град", "Дар", "Мир", "Рад", "Свят", "Яр"],
|
||||
["", "о", "е", "и", "а"],
|
||||
["ан", "ен", "ий", "ор", "слав", "мир", "дан", "бор", "гор", "вит"],
|
||||
),
|
||||
1: (
|
||||
["Аль", "Бож", "Вес", "Дар", "Злат", "Лад", "Люб", "Мил", "Рад", "Яр"],
|
||||
["", "о", "е", "и", "а"],
|
||||
["ана", "ена", "ина", "ира", "исса", "лава", "мира", "яна", "ея", "ора"],
|
||||
),
|
||||
2: (
|
||||
["Блик", "Винт", "Гайк", "Жуж", "Искр", "Клап", "Порш", "Рыч", "Тик", "Шпун"],
|
||||
["", "о", "е", "и", "а"],
|
||||
["ик", "ак", "ун", "ил", "ер", "орт", "ен", "ос", "ин", "ей"],
|
||||
),
|
||||
3: (
|
||||
["Блик", "Винт", "Гайк", "Жуж", "Искр", "Кноп", "Пруж", "Тик", "Шпул", "Шестер"],
|
||||
["", "о", "е", "и", "а"],
|
||||
["ика", "ана", "инка", "елла", "ира", "ина", "етта", "исса", "уна", "ея"],
|
||||
),
|
||||
4: (
|
||||
["Бран", "Бром", "Гим", "Дор", "Каз", "Краг", "Мор", "Тор", "Фар", "Хар"],
|
||||
["", "о", "е", "и", "а"],
|
||||
["дин", "гар", "грим", "мунд", "бор", "рик", "вар", "драм", "гран", "ор"],
|
||||
),
|
||||
5: (
|
||||
["Бран", "Бром", "Гим", "Дор", "Каз", "Краг", "Мор", "Тор", "Фар", "Хель"],
|
||||
["", "о", "е", "и", "а"],
|
||||
["дина", "гара", "гримма", "мунда", "бора", "рика", "вара", "дра", "грана", "ина"],
|
||||
),
|
||||
6: (
|
||||
["Аэл", "Илл", "Лиар", "Мэл", "Ним", "Саэл", "Тал", "Фаэр", "Эли", "Яв"],
|
||||
["", "о", "е", "и", "а"],
|
||||
["дор", "рион", "лас", "тир", "ниэл", "вен", "рон", "лир", "тис", "эль"],
|
||||
),
|
||||
7: (
|
||||
["Аэл", "Илл", "Лиа", "Мэл", "Ниа", "Саэл", "Тали", "Фаэ", "Эли", "Яви"],
|
||||
["", "о", "е", "и", "а"],
|
||||
["дора", "риэль", "ласса", "тира", "ниэль", "вена", "рона", "лира", "тисса", "эля"],
|
||||
),
|
||||
8: (
|
||||
["Ака", "Ару", "Ваа", "Ири", "Каа", "Наэ", "Оро", "Таа", "Эре", "Яна"],
|
||||
["", "л", "р", "н", "м"],
|
||||
["дор", "ниус", "мар", "рион", "тар", "вен", "мон", "дар", "рос", "эль"],
|
||||
),
|
||||
9: (
|
||||
["Ака", "Ару", "Ваа", "Ири", "Каа", "Наэ", "Оро", "Таа", "Эре", "Яна"],
|
||||
["", "л", "р", "н", "м"],
|
||||
["дора", "ния", "мара", "риэль", "тара", "вена", "мона", "дара", "роса", "эла"],
|
||||
),
|
||||
10: (
|
||||
["Гар", "Гром", "Драк", "Карг", "Мок", "Рок", "Тар", "Ург", "Хар", "Зуг"],
|
||||
["", "о", "а", "у", "и"],
|
||||
["аш", "мак", "гар", "док", "рат", "гул", "нак", "рок", "тар", "зог"],
|
||||
),
|
||||
11: (
|
||||
["Гра", "Гром", "Дра", "Кар", "Мо", "Ро", "Тар", "Ур", "Хар", "Зу"],
|
||||
["", "о", "а", "у", "и"],
|
||||
["ша", "мака", "гара", "дока", "рата", "гула", "нака", "рока", "тара", "зога"],
|
||||
),
|
||||
12: (
|
||||
["Вол", "Джин", "Зан", "Зул", "Каз", "Раз", "Сэн", "Таз", "Хак", "Ям"],
|
||||
["", "а", "о", "и", "у"],
|
||||
["джи", "зар", "кан", "мон", "рак", "тал", "хан", "зин", "джо", "мар"],
|
||||
),
|
||||
13: (
|
||||
["Вола", "Джи", "Зана", "Зула", "Каза", "Раза", "Сэна", "Таза", "Хака", "Яма"],
|
||||
["", "а", "о", "и", "у"],
|
||||
["джи", "зара", "кана", "мона", "рака", "тала", "хана", "зина", "джа", "мара"],
|
||||
),
|
||||
14: (
|
||||
["Гром", "Кам", "Коп", "Неб", "Рог", "Степ", "Туч", "Шрам", "Ветр", "Биз"],
|
||||
["", "о", "е", "и", "а"],
|
||||
["орог", "оступ", "ебык", "огрив", "окоп", "оступ", "ешаг", "арог", "овет", "он"],
|
||||
),
|
||||
15: (
|
||||
["Гром", "Кам", "Коп", "Неб", "Рог", "Степ", "Туч", "Шрам", "Ветр", "Биз"],
|
||||
["", "о", "е", "и", "а"],
|
||||
["орогая", "оступа", "ебыка", "огрива", "окопа", "ешага", "арога", "овета", "она", "ани"],
|
||||
),
|
||||
16: (
|
||||
["Алер", "Вей", "Каэл", "Кел", "Лор", "Сал", "Тал", "Фен", "Эль", "Эрен"],
|
||||
["", "а", "е", "и", "о"],
|
||||
["ан", "ион", "дир", "лас", "рен", "тос", "вар", "лен", "рис", "эль"],
|
||||
),
|
||||
17: (
|
||||
["Але", "Вей", "Каэ", "Кели", "Лори", "Сали", "Тали", "Фе", "Эли", "Эре"],
|
||||
["", "а", "е", "и", "о"],
|
||||
["ана", "иэль", "дира", "ласса", "рена", "тоса", "вара", "лена", "риса", "эля"],
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
GUILD_PREFIXES = [
|
||||
"Стражи", "Клинки", "Герои", "Хранители", "Вестники", "Воины",
|
||||
"Дети", "Сыны", "Дочери", "Рыцари", "Искатели", "Защитники",
|
||||
]
|
||||
GUILD_OBJECTS = [
|
||||
"Рассвета", "Заката", "Севера", "Юга", "Азерота", "Калимдора",
|
||||
"Бури", "Пламени", "Льда", "Теней", "Луны", "Солнца",
|
||||
"Чести", "Славы", "Свободы", "Надежды", "Дракона", "Ворона",
|
||||
]
|
||||
GUILD_ADJECTIVES = [
|
||||
"Лунный", "Стальной", "Серебряный", "Золотой", "Алый", "Сумрачный",
|
||||
"Ночной", "Северный", "Вечный", "Тихий", "Грозовой", "Огненный",
|
||||
]
|
||||
GUILD_NOUNS = [
|
||||
"Дозор", "Легион", "Орден", "Союз", "Круг", "Щит",
|
||||
"Клинок", "Рубеж", "Рассвет", "Оплот", "Поход", "Ветер",
|
||||
]
|
||||
|
||||
|
||||
def valid_name(value: str) -> bool:
|
||||
return (
|
||||
2 <= len(value) <= 12
|
||||
and value.isalpha()
|
||||
and all("А" <= char <= "я" or char in "Ёё" for char in value)
|
||||
and not any(value[index] == value[index - 1] == value[index - 2] for index in range(2, len(value)))
|
||||
)
|
||||
|
||||
|
||||
TRANSLITERATION = {
|
||||
"shch": "щ",
|
||||
"sch": "щ",
|
||||
"zh": "ж",
|
||||
"kh": "х",
|
||||
"ts": "ц",
|
||||
"ch": "ч",
|
||||
"sh": "ш",
|
||||
"th": "т",
|
||||
"ph": "ф",
|
||||
"qu": "кв",
|
||||
"ck": "к",
|
||||
"ya": "я",
|
||||
"yo": "ё",
|
||||
"yu": "ю",
|
||||
"a": "а",
|
||||
"b": "б",
|
||||
"c": "к",
|
||||
"d": "д",
|
||||
"e": "е",
|
||||
"f": "ф",
|
||||
"g": "г",
|
||||
"h": "х",
|
||||
"i": "и",
|
||||
"j": "дж",
|
||||
"k": "к",
|
||||
"l": "л",
|
||||
"m": "м",
|
||||
"n": "н",
|
||||
"o": "о",
|
||||
"p": "п",
|
||||
"q": "к",
|
||||
"r": "р",
|
||||
"s": "с",
|
||||
"t": "т",
|
||||
"u": "у",
|
||||
"v": "в",
|
||||
"w": "в",
|
||||
"x": "кс",
|
||||
"y": "й",
|
||||
"z": "з",
|
||||
}
|
||||
|
||||
|
||||
def transliterate_name(value: str) -> str:
|
||||
source = value.casefold()
|
||||
translated: list[str] = []
|
||||
index = 0
|
||||
keys = sorted(TRANSLITERATION, key=len, reverse=True)
|
||||
|
||||
while index < len(source):
|
||||
for key in keys:
|
||||
if source.startswith(key, index):
|
||||
translated.append(TRANSLITERATION[key])
|
||||
index += len(key)
|
||||
break
|
||||
else:
|
||||
return ""
|
||||
|
||||
return "".join(translated).capitalize()
|
||||
|
||||
|
||||
def mysql_unicode_ci_key(value: str) -> str:
|
||||
"""Approximate the equivalences used by the target utf8mb4_unicode_ci index."""
|
||||
return value.casefold().replace("ё", "е").replace("й", "и")
|
||||
|
||||
|
||||
def generate_character_names() -> list[tuple[str, int]]:
|
||||
used: set[str] = set()
|
||||
result: list[tuple[str, int]] = []
|
||||
by_category: dict[int, list[str]] = {category: [] for category in range(18)}
|
||||
row_pattern = re.compile(r"^\(\d+,'([^']+)',(\d+)\)[,;]$")
|
||||
|
||||
for line in UPSTREAM_NAMES.read_text(encoding="utf-8").splitlines():
|
||||
match = row_pattern.match(line.strip())
|
||||
if not match:
|
||||
continue
|
||||
|
||||
original, category_text = match.groups()
|
||||
category = int(category_text)
|
||||
if category not in by_category or len(by_category[category]) >= NAMES_PER_CATEGORY:
|
||||
continue
|
||||
|
||||
candidate = transliterate_name(original)
|
||||
candidate_key = mysql_unicode_ci_key(candidate)
|
||||
if not valid_name(candidate) or candidate_key in used:
|
||||
continue
|
||||
|
||||
used.add(candidate_key)
|
||||
by_category[category].append(candidate)
|
||||
|
||||
for category, category_names in by_category.items():
|
||||
if len(category_names) != NAMES_PER_CATEGORY:
|
||||
raise RuntimeError(
|
||||
f"Category {category} produced {len(category_names)} names, expected {NAMES_PER_CATEGORY}"
|
||||
)
|
||||
result.extend((name, category) for name in category_names)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def generate_guild_names() -> list[str]:
|
||||
candidates = [
|
||||
f"{prefix} {obj}"
|
||||
for prefix, obj in itertools.product(GUILD_PREFIXES, GUILD_OBJECTS)
|
||||
]
|
||||
candidates.extend(
|
||||
f"{adjective} {noun}"
|
||||
for adjective, noun in itertools.product(GUILD_ADJECTIVES, GUILD_NOUNS)
|
||||
)
|
||||
return list(dict.fromkeys(name for name in candidates if len(name) <= 24))
|
||||
|
||||
|
||||
def sql_quote(value: str) -> str:
|
||||
return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
names = generate_character_names()
|
||||
guild_names = generate_guild_names()
|
||||
|
||||
lines = [
|
||||
"-- MoonWell static Cyrillic name pools for mod-playerbots.",
|
||||
"-- Generated by tools/generate-playerbots-ru-names.py.",
|
||||
"SET NAMES utf8mb4;",
|
||||
"",
|
||||
"CREATE TABLE IF NOT EXISTS `playerbots_names_ru` (",
|
||||
" `name_id` INT UNSIGNED NOT NULL AUTO_INCREMENT,",
|
||||
" `name` VARCHAR(24) NOT NULL,",
|
||||
" `gender` TINYINT UNSIGNED NOT NULL,",
|
||||
" PRIMARY KEY (`name_id`),",
|
||||
" KEY `idx_playerbots_names_ru_name` (`name`)",
|
||||
") ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;",
|
||||
"",
|
||||
"TRUNCATE TABLE `playerbots_names_ru`;",
|
||||
"INSERT INTO `playerbots_names_ru` (`name`, `gender`) VALUES",
|
||||
]
|
||||
lines.extend(
|
||||
f"({sql_quote(name)}, {category}){',' if index + 1 < len(names) else ';'}"
|
||||
for index, (name, category) in enumerate(names)
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"CREATE TABLE IF NOT EXISTS `playerbots_guild_names_ru` (",
|
||||
" `name_id` INT UNSIGNED NOT NULL AUTO_INCREMENT,",
|
||||
" `name` VARCHAR(24) NOT NULL,",
|
||||
" PRIMARY KEY (`name_id`),",
|
||||
" UNIQUE KEY `uq_playerbots_guild_names_ru_name` (`name`)",
|
||||
") ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;",
|
||||
"",
|
||||
"TRUNCATE TABLE `playerbots_guild_names_ru`;",
|
||||
"INSERT INTO `playerbots_guild_names_ru` (`name`) VALUES",
|
||||
]
|
||||
)
|
||||
lines.extend(
|
||||
f"({sql_quote(name)}){',' if index + 1 < len(guild_names) else ';'}"
|
||||
for index, name in enumerate(guild_names)
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
OUTPUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
OUTPUT.write_text("\n".join(lines), encoding="utf-8")
|
||||
print(f"Wrote {len(names)} character names and {len(guild_names)} guild names to {OUTPUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,220 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""Generate the final static ruRU update for untranslated playerbot texts.
|
||||
|
||||
Run this after the playerbots database is fully updated. The generation-only
|
||||
dependencies are intentionally not part of the server runtime:
|
||||
|
||||
pip install torch transformers sentencepiece sacremoses
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUTPUT = (
|
||||
ROOT
|
||||
/ "modules/mod-playerbots/data/sql/playerbots/updates/"
|
||||
/ "2026_07_30_00_ai_playerbot_russian_texts.sql"
|
||||
)
|
||||
MODEL_NAME = os.environ.get(
|
||||
"PLAYERBOTS_RU_TRANSLATION_MODEL",
|
||||
"facebook/nllb-200-distilled-600M",
|
||||
)
|
||||
PLACEHOLDER_RE = re.compile(r"%[A-Za-z_][A-Za-z0-9_]*|<[^>]+>")
|
||||
|
||||
|
||||
def load_untranslated_rows() -> list[tuple[int, str, str]]:
|
||||
sql = (
|
||||
"SELECT id,HEX(name),HEX(text) "
|
||||
"FROM ai_playerbot_texts WHERE LENGTH(text_loc8)=0 ORDER BY id"
|
||||
)
|
||||
command = [
|
||||
"docker",
|
||||
"compose",
|
||||
"exec",
|
||||
"-T",
|
||||
"ac-database",
|
||||
"bash",
|
||||
"-lc",
|
||||
f'mysql -N -s -uroot -p"$MYSQL_ROOT_PASSWORD" acore_playerbots -e "{sql}" 2>/dev/null',
|
||||
]
|
||||
output = subprocess.check_output(command, cwd=ROOT, text=True)
|
||||
rows: list[tuple[int, str, str]] = []
|
||||
|
||||
for line in output.splitlines():
|
||||
parts = line.split("\t")
|
||||
if len(parts) != 3:
|
||||
continue
|
||||
row_id = int(parts[0])
|
||||
name = bytes.fromhex(parts[1]).decode("utf-8")
|
||||
text = bytes.fromhex(parts[2]).decode("utf-8").replace("\ufffd", "'")
|
||||
rows.append((row_id, name, text))
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def sql_quote(value: str) -> str:
|
||||
return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'"
|
||||
|
||||
|
||||
def prepare_source(text: str) -> str:
|
||||
replacements = {
|
||||
"I’m": "I am",
|
||||
"I’ll": "I will",
|
||||
"I’ve": "I have",
|
||||
"won’t": "will not",
|
||||
"can’t": "cannot",
|
||||
"don’t": "do not",
|
||||
"Let’s": "Let us",
|
||||
"Just killed %victim_name": "I have just defeated %victim_name",
|
||||
"%victim_name was too easy": "Defeating %victim_name was too easy",
|
||||
"More %faction rep": "More reputation with %faction",
|
||||
"grinding %faction rep": "earning reputation with %faction",
|
||||
"farm %category": "collect %category",
|
||||
"farming %category": "collecting %category",
|
||||
"looting": "collecting loot",
|
||||
"hit me up": "message me",
|
||||
"Hit me up": "Message me",
|
||||
"hit level": "reached level",
|
||||
"Hit level": "Reached level",
|
||||
"turned it in": "handed it in",
|
||||
"over nothing": "rather than getting nothing",
|
||||
"Any takers": "Who wants",
|
||||
"a solid ": "a good ",
|
||||
"A solid ": "A good ",
|
||||
"great deal": "good price",
|
||||
"smack talk": "insults",
|
||||
}
|
||||
prepared = text
|
||||
for source, target in replacements.items():
|
||||
prepared = prepared.replace(source, target)
|
||||
prepared = re.sub(r"\brep\b", "reputation", prepared, flags=re.IGNORECASE)
|
||||
prepared = re.sub(r"\bgrinding\b", "earning", prepared, flags=re.IGNORECASE)
|
||||
prepared = re.sub(r"\bgrind\b", "earn", prepared, flags=re.IGNORECASE)
|
||||
prepared = re.sub(r"\baggro\b", "attract enemies", prepared, flags=re.IGNORECASE)
|
||||
return prepared
|
||||
|
||||
|
||||
def translate(rows: list[tuple[int, str, str]]) -> list[tuple[int, str, str, str]]:
|
||||
is_nllb = "nllb" in MODEL_NAME.casefold()
|
||||
tokenizer_options = {"src_lang": "eng_Latn"} if is_nllb else {}
|
||||
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, **tokenizer_options)
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
dtype = torch.float16 if device.type == "cuda" else torch.float32
|
||||
model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME, torch_dtype=dtype).to(device)
|
||||
batch_size = 48 if device.type == "cuda" else 12
|
||||
result: list[tuple[int, str, str, str]] = []
|
||||
|
||||
def generate_texts(texts: list[str]) -> list[str]:
|
||||
encoded = tokenizer(
|
||||
texts,
|
||||
return_tensors="pt",
|
||||
padding=True,
|
||||
truncation=True,
|
||||
max_length=256,
|
||||
)
|
||||
encoded = {key: value.to(device) for key, value in encoded.items()}
|
||||
generation_options = {
|
||||
"max_new_tokens": 256,
|
||||
"num_beams": 4,
|
||||
"early_stopping": True,
|
||||
}
|
||||
if is_nllb:
|
||||
generation_options["forced_bos_token_id"] = tokenizer.convert_tokens_to_ids("rus_Cyrl")
|
||||
generated = model.generate(
|
||||
**encoded,
|
||||
**generation_options,
|
||||
)
|
||||
return tokenizer.batch_decode(generated, skip_special_tokens=True)
|
||||
|
||||
def translate_with_segments(source: str) -> str:
|
||||
parts = re.split(f"({PLACEHOLDER_RE.pattern})", source)
|
||||
jobs: list[tuple[int, str, str, str]] = []
|
||||
for part_index, part in enumerate(parts):
|
||||
if not part or PLACEHOLDER_RE.fullmatch(part):
|
||||
continue
|
||||
alpha_indexes = [index for index, char in enumerate(part) if char.isalpha()]
|
||||
if not alpha_indexes:
|
||||
continue
|
||||
first_alpha = alpha_indexes[0]
|
||||
last_alpha = alpha_indexes[-1]
|
||||
jobs.append(
|
||||
(
|
||||
part_index,
|
||||
part[:first_alpha],
|
||||
prepare_source(part[first_alpha : last_alpha + 1]),
|
||||
part[last_alpha + 1 :],
|
||||
)
|
||||
)
|
||||
|
||||
translated_parts = generate_texts([core for _index, _leading, core, _trailing in jobs])
|
||||
for (part_index, leading, _core, trailing), target in zip(jobs, translated_parts):
|
||||
parts[part_index] = leading + target + trailing
|
||||
return "".join(parts)
|
||||
|
||||
for start in range(0, len(rows), batch_size):
|
||||
batch = rows[start : start + batch_size]
|
||||
translated = generate_texts(
|
||||
[prepare_source(source) for _row_id, _name, source in batch]
|
||||
)
|
||||
|
||||
for (row_id, name, source), target in zip(batch, translated):
|
||||
source_placeholders = Counter(PLACEHOLDER_RE.findall(source))
|
||||
target_placeholders = Counter(PLACEHOLDER_RE.findall(target))
|
||||
if source_placeholders != target_placeholders:
|
||||
target = translate_with_segments(source)
|
||||
target_placeholders = Counter(PLACEHOLDER_RE.findall(target))
|
||||
if source_placeholders != target_placeholders:
|
||||
raise RuntimeError(
|
||||
f"Placeholder mismatch for id={row_id}: "
|
||||
f"{source_placeholders} != {target_placeholders}"
|
||||
)
|
||||
result.append((row_id, name, source, target))
|
||||
|
||||
print(
|
||||
f"Translated {min(start + batch_size, len(rows))}/{len(rows)} rows",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def write_sql(rows: list[tuple[int, str, str, str]]) -> None:
|
||||
lines = [
|
||||
"-- MoonWell static Russian localization for mod-playerbots.",
|
||||
"-- Generated after all upstream playerbots text migrations.",
|
||||
"-- Placeholders are validated by tools/generate-playerbots-ru-texts.py.",
|
||||
"SET NAMES utf8mb4;",
|
||||
"",
|
||||
"START TRANSACTION;",
|
||||
]
|
||||
for row_id, name, _source, target in rows:
|
||||
lines.append(
|
||||
"UPDATE `ai_playerbot_texts` "
|
||||
f"SET `text_loc8`={sql_quote(target)} "
|
||||
f"WHERE `id`={row_id} AND `name`={sql_quote(name)} AND LENGTH(`text_loc8`)=0;"
|
||||
)
|
||||
lines.extend(["COMMIT;", ""])
|
||||
OUTPUT.write_text("\n".join(lines), encoding="utf-8")
|
||||
print(f"Wrote {len(rows)} translations to {OUTPUT}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rows = load_untranslated_rows()
|
||||
if not rows:
|
||||
raise SystemExit("No untranslated playerbot text rows found")
|
||||
write_sql(translate(rows))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user