221 lines
7.8 KiB
Python
221 lines
7.8 KiB
Python
#!/usr/bin/env python3
|
||
|
||
"""Generate the final static ruRU update for untranslated playerbot texts.
|
||
|
||
Run this after the playerbots database is fully updated. The generation-only
|
||
dependencies are intentionally not part of the server runtime:
|
||
|
||
pip install torch transformers sentencepiece sacremoses
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
import re
|
||
import subprocess
|
||
from collections import Counter
|
||
from pathlib import Path
|
||
|
||
import torch
|
||
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
|
||
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
OUTPUT = (
|
||
ROOT
|
||
/ "modules/mod-playerbots/data/sql/playerbots/updates/"
|
||
/ "2026_07_30_00_ai_playerbot_russian_texts.sql"
|
||
)
|
||
MODEL_NAME = os.environ.get(
|
||
"PLAYERBOTS_RU_TRANSLATION_MODEL",
|
||
"facebook/nllb-200-distilled-600M",
|
||
)
|
||
PLACEHOLDER_RE = re.compile(r"%[A-Za-z_][A-Za-z0-9_]*|<[^>]+>")
|
||
|
||
|
||
def load_untranslated_rows() -> list[tuple[int, str, str]]:
|
||
sql = (
|
||
"SELECT id,HEX(name),HEX(text) "
|
||
"FROM ai_playerbot_texts WHERE LENGTH(text_loc8)=0 ORDER BY id"
|
||
)
|
||
command = [
|
||
"docker",
|
||
"compose",
|
||
"exec",
|
||
"-T",
|
||
"ac-database",
|
||
"bash",
|
||
"-lc",
|
||
f'mysql -N -s -uroot -p"$MYSQL_ROOT_PASSWORD" acore_playerbots -e "{sql}" 2>/dev/null',
|
||
]
|
||
output = subprocess.check_output(command, cwd=ROOT, text=True)
|
||
rows: list[tuple[int, str, str]] = []
|
||
|
||
for line in output.splitlines():
|
||
parts = line.split("\t")
|
||
if len(parts) != 3:
|
||
continue
|
||
row_id = int(parts[0])
|
||
name = bytes.fromhex(parts[1]).decode("utf-8")
|
||
text = bytes.fromhex(parts[2]).decode("utf-8").replace("\ufffd", "'")
|
||
rows.append((row_id, name, text))
|
||
|
||
return rows
|
||
|
||
|
||
def sql_quote(value: str) -> str:
|
||
return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'"
|
||
|
||
|
||
def prepare_source(text: str) -> str:
|
||
replacements = {
|
||
"I’m": "I am",
|
||
"I’ll": "I will",
|
||
"I’ve": "I have",
|
||
"won’t": "will not",
|
||
"can’t": "cannot",
|
||
"don’t": "do not",
|
||
"Let’s": "Let us",
|
||
"Just killed %victim_name": "I have just defeated %victim_name",
|
||
"%victim_name was too easy": "Defeating %victim_name was too easy",
|
||
"More %faction rep": "More reputation with %faction",
|
||
"grinding %faction rep": "earning reputation with %faction",
|
||
"farm %category": "collect %category",
|
||
"farming %category": "collecting %category",
|
||
"looting": "collecting loot",
|
||
"hit me up": "message me",
|
||
"Hit me up": "Message me",
|
||
"hit level": "reached level",
|
||
"Hit level": "Reached level",
|
||
"turned it in": "handed it in",
|
||
"over nothing": "rather than getting nothing",
|
||
"Any takers": "Who wants",
|
||
"a solid ": "a good ",
|
||
"A solid ": "A good ",
|
||
"great deal": "good price",
|
||
"smack talk": "insults",
|
||
}
|
||
prepared = text
|
||
for source, target in replacements.items():
|
||
prepared = prepared.replace(source, target)
|
||
prepared = re.sub(r"\brep\b", "reputation", prepared, flags=re.IGNORECASE)
|
||
prepared = re.sub(r"\bgrinding\b", "earning", prepared, flags=re.IGNORECASE)
|
||
prepared = re.sub(r"\bgrind\b", "earn", prepared, flags=re.IGNORECASE)
|
||
prepared = re.sub(r"\baggro\b", "attract enemies", prepared, flags=re.IGNORECASE)
|
||
return prepared
|
||
|
||
|
||
def translate(rows: list[tuple[int, str, str]]) -> list[tuple[int, str, str, str]]:
|
||
is_nllb = "nllb" in MODEL_NAME.casefold()
|
||
tokenizer_options = {"src_lang": "eng_Latn"} if is_nllb else {}
|
||
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, **tokenizer_options)
|
||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||
dtype = torch.float16 if device.type == "cuda" else torch.float32
|
||
model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME, torch_dtype=dtype).to(device)
|
||
batch_size = 48 if device.type == "cuda" else 12
|
||
result: list[tuple[int, str, str, str]] = []
|
||
|
||
def generate_texts(texts: list[str]) -> list[str]:
|
||
encoded = tokenizer(
|
||
texts,
|
||
return_tensors="pt",
|
||
padding=True,
|
||
truncation=True,
|
||
max_length=256,
|
||
)
|
||
encoded = {key: value.to(device) for key, value in encoded.items()}
|
||
generation_options = {
|
||
"max_new_tokens": 256,
|
||
"num_beams": 4,
|
||
"early_stopping": True,
|
||
}
|
||
if is_nllb:
|
||
generation_options["forced_bos_token_id"] = tokenizer.convert_tokens_to_ids("rus_Cyrl")
|
||
generated = model.generate(
|
||
**encoded,
|
||
**generation_options,
|
||
)
|
||
return tokenizer.batch_decode(generated, skip_special_tokens=True)
|
||
|
||
def translate_with_segments(source: str) -> str:
|
||
parts = re.split(f"({PLACEHOLDER_RE.pattern})", source)
|
||
jobs: list[tuple[int, str, str, str]] = []
|
||
for part_index, part in enumerate(parts):
|
||
if not part or PLACEHOLDER_RE.fullmatch(part):
|
||
continue
|
||
alpha_indexes = [index for index, char in enumerate(part) if char.isalpha()]
|
||
if not alpha_indexes:
|
||
continue
|
||
first_alpha = alpha_indexes[0]
|
||
last_alpha = alpha_indexes[-1]
|
||
jobs.append(
|
||
(
|
||
part_index,
|
||
part[:first_alpha],
|
||
prepare_source(part[first_alpha : last_alpha + 1]),
|
||
part[last_alpha + 1 :],
|
||
)
|
||
)
|
||
|
||
translated_parts = generate_texts([core for _index, _leading, core, _trailing in jobs])
|
||
for (part_index, leading, _core, trailing), target in zip(jobs, translated_parts):
|
||
parts[part_index] = leading + target + trailing
|
||
return "".join(parts)
|
||
|
||
for start in range(0, len(rows), batch_size):
|
||
batch = rows[start : start + batch_size]
|
||
translated = generate_texts(
|
||
[prepare_source(source) for _row_id, _name, source in batch]
|
||
)
|
||
|
||
for (row_id, name, source), target in zip(batch, translated):
|
||
source_placeholders = Counter(PLACEHOLDER_RE.findall(source))
|
||
target_placeholders = Counter(PLACEHOLDER_RE.findall(target))
|
||
if source_placeholders != target_placeholders:
|
||
target = translate_with_segments(source)
|
||
target_placeholders = Counter(PLACEHOLDER_RE.findall(target))
|
||
if source_placeholders != target_placeholders:
|
||
raise RuntimeError(
|
||
f"Placeholder mismatch for id={row_id}: "
|
||
f"{source_placeholders} != {target_placeholders}"
|
||
)
|
||
result.append((row_id, name, source, target))
|
||
|
||
print(
|
||
f"Translated {min(start + batch_size, len(rows))}/{len(rows)} rows",
|
||
flush=True,
|
||
)
|
||
|
||
return result
|
||
|
||
|
||
def write_sql(rows: list[tuple[int, str, str, str]]) -> None:
|
||
lines = [
|
||
"-- MoonWell static Russian localization for mod-playerbots.",
|
||
"-- Generated after all upstream playerbots text migrations.",
|
||
"-- Placeholders are validated by tools/generate-playerbots-ru-texts.py.",
|
||
"SET NAMES utf8mb4;",
|
||
"",
|
||
"START TRANSACTION;",
|
||
]
|
||
for row_id, name, _source, target in rows:
|
||
lines.append(
|
||
"UPDATE `ai_playerbot_texts` "
|
||
f"SET `text_loc8`={sql_quote(target)} "
|
||
f"WHERE `id`={row_id} AND `name`={sql_quote(name)} AND LENGTH(`text_loc8`)=0;"
|
||
)
|
||
lines.extend(["COMMIT;", ""])
|
||
OUTPUT.write_text("\n".join(lines), encoding="utf-8")
|
||
print(f"Wrote {len(rows)} translations to {OUTPUT}")
|
||
|
||
|
||
def main() -> None:
|
||
rows = load_untranslated_rows()
|
||
if not rows:
|
||
raise SystemExit("No untranslated playerbot text rows found")
|
||
write_sql(translate(rows))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|