боты русифицированы
This commit is contained in:
@@ -0,0 +1,220 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""Generate the final static ruRU update for untranslated playerbot texts.
|
||||
|
||||
Run this after the playerbots database is fully updated. The generation-only
|
||||
dependencies are intentionally not part of the server runtime:
|
||||
|
||||
pip install torch transformers sentencepiece sacremoses
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUTPUT = (
|
||||
ROOT
|
||||
/ "modules/mod-playerbots/data/sql/playerbots/updates/"
|
||||
/ "2026_07_30_00_ai_playerbot_russian_texts.sql"
|
||||
)
|
||||
MODEL_NAME = os.environ.get(
|
||||
"PLAYERBOTS_RU_TRANSLATION_MODEL",
|
||||
"facebook/nllb-200-distilled-600M",
|
||||
)
|
||||
PLACEHOLDER_RE = re.compile(r"%[A-Za-z_][A-Za-z0-9_]*|<[^>]+>")
|
||||
|
||||
|
||||
def load_untranslated_rows() -> list[tuple[int, str, str]]:
|
||||
sql = (
|
||||
"SELECT id,HEX(name),HEX(text) "
|
||||
"FROM ai_playerbot_texts WHERE LENGTH(text_loc8)=0 ORDER BY id"
|
||||
)
|
||||
command = [
|
||||
"docker",
|
||||
"compose",
|
||||
"exec",
|
||||
"-T",
|
||||
"ac-database",
|
||||
"bash",
|
||||
"-lc",
|
||||
f'mysql -N -s -uroot -p"$MYSQL_ROOT_PASSWORD" acore_playerbots -e "{sql}" 2>/dev/null',
|
||||
]
|
||||
output = subprocess.check_output(command, cwd=ROOT, text=True)
|
||||
rows: list[tuple[int, str, str]] = []
|
||||
|
||||
for line in output.splitlines():
|
||||
parts = line.split("\t")
|
||||
if len(parts) != 3:
|
||||
continue
|
||||
row_id = int(parts[0])
|
||||
name = bytes.fromhex(parts[1]).decode("utf-8")
|
||||
text = bytes.fromhex(parts[2]).decode("utf-8").replace("\ufffd", "'")
|
||||
rows.append((row_id, name, text))
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def sql_quote(value: str) -> str:
|
||||
return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'"
|
||||
|
||||
|
||||
def prepare_source(text: str) -> str:
|
||||
replacements = {
|
||||
"I’m": "I am",
|
||||
"I’ll": "I will",
|
||||
"I’ve": "I have",
|
||||
"won’t": "will not",
|
||||
"can’t": "cannot",
|
||||
"don’t": "do not",
|
||||
"Let’s": "Let us",
|
||||
"Just killed %victim_name": "I have just defeated %victim_name",
|
||||
"%victim_name was too easy": "Defeating %victim_name was too easy",
|
||||
"More %faction rep": "More reputation with %faction",
|
||||
"grinding %faction rep": "earning reputation with %faction",
|
||||
"farm %category": "collect %category",
|
||||
"farming %category": "collecting %category",
|
||||
"looting": "collecting loot",
|
||||
"hit me up": "message me",
|
||||
"Hit me up": "Message me",
|
||||
"hit level": "reached level",
|
||||
"Hit level": "Reached level",
|
||||
"turned it in": "handed it in",
|
||||
"over nothing": "rather than getting nothing",
|
||||
"Any takers": "Who wants",
|
||||
"a solid ": "a good ",
|
||||
"A solid ": "A good ",
|
||||
"great deal": "good price",
|
||||
"smack talk": "insults",
|
||||
}
|
||||
prepared = text
|
||||
for source, target in replacements.items():
|
||||
prepared = prepared.replace(source, target)
|
||||
prepared = re.sub(r"\brep\b", "reputation", prepared, flags=re.IGNORECASE)
|
||||
prepared = re.sub(r"\bgrinding\b", "earning", prepared, flags=re.IGNORECASE)
|
||||
prepared = re.sub(r"\bgrind\b", "earn", prepared, flags=re.IGNORECASE)
|
||||
prepared = re.sub(r"\baggro\b", "attract enemies", prepared, flags=re.IGNORECASE)
|
||||
return prepared
|
||||
|
||||
|
||||
def translate(rows: list[tuple[int, str, str]]) -> list[tuple[int, str, str, str]]:
|
||||
is_nllb = "nllb" in MODEL_NAME.casefold()
|
||||
tokenizer_options = {"src_lang": "eng_Latn"} if is_nllb else {}
|
||||
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, **tokenizer_options)
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
dtype = torch.float16 if device.type == "cuda" else torch.float32
|
||||
model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME, torch_dtype=dtype).to(device)
|
||||
batch_size = 48 if device.type == "cuda" else 12
|
||||
result: list[tuple[int, str, str, str]] = []
|
||||
|
||||
def generate_texts(texts: list[str]) -> list[str]:
|
||||
encoded = tokenizer(
|
||||
texts,
|
||||
return_tensors="pt",
|
||||
padding=True,
|
||||
truncation=True,
|
||||
max_length=256,
|
||||
)
|
||||
encoded = {key: value.to(device) for key, value in encoded.items()}
|
||||
generation_options = {
|
||||
"max_new_tokens": 256,
|
||||
"num_beams": 4,
|
||||
"early_stopping": True,
|
||||
}
|
||||
if is_nllb:
|
||||
generation_options["forced_bos_token_id"] = tokenizer.convert_tokens_to_ids("rus_Cyrl")
|
||||
generated = model.generate(
|
||||
**encoded,
|
||||
**generation_options,
|
||||
)
|
||||
return tokenizer.batch_decode(generated, skip_special_tokens=True)
|
||||
|
||||
def translate_with_segments(source: str) -> str:
|
||||
parts = re.split(f"({PLACEHOLDER_RE.pattern})", source)
|
||||
jobs: list[tuple[int, str, str, str]] = []
|
||||
for part_index, part in enumerate(parts):
|
||||
if not part or PLACEHOLDER_RE.fullmatch(part):
|
||||
continue
|
||||
alpha_indexes = [index for index, char in enumerate(part) if char.isalpha()]
|
||||
if not alpha_indexes:
|
||||
continue
|
||||
first_alpha = alpha_indexes[0]
|
||||
last_alpha = alpha_indexes[-1]
|
||||
jobs.append(
|
||||
(
|
||||
part_index,
|
||||
part[:first_alpha],
|
||||
prepare_source(part[first_alpha : last_alpha + 1]),
|
||||
part[last_alpha + 1 :],
|
||||
)
|
||||
)
|
||||
|
||||
translated_parts = generate_texts([core for _index, _leading, core, _trailing in jobs])
|
||||
for (part_index, leading, _core, trailing), target in zip(jobs, translated_parts):
|
||||
parts[part_index] = leading + target + trailing
|
||||
return "".join(parts)
|
||||
|
||||
for start in range(0, len(rows), batch_size):
|
||||
batch = rows[start : start + batch_size]
|
||||
translated = generate_texts(
|
||||
[prepare_source(source) for _row_id, _name, source in batch]
|
||||
)
|
||||
|
||||
for (row_id, name, source), target in zip(batch, translated):
|
||||
source_placeholders = Counter(PLACEHOLDER_RE.findall(source))
|
||||
target_placeholders = Counter(PLACEHOLDER_RE.findall(target))
|
||||
if source_placeholders != target_placeholders:
|
||||
target = translate_with_segments(source)
|
||||
target_placeholders = Counter(PLACEHOLDER_RE.findall(target))
|
||||
if source_placeholders != target_placeholders:
|
||||
raise RuntimeError(
|
||||
f"Placeholder mismatch for id={row_id}: "
|
||||
f"{source_placeholders} != {target_placeholders}"
|
||||
)
|
||||
result.append((row_id, name, source, target))
|
||||
|
||||
print(
|
||||
f"Translated {min(start + batch_size, len(rows))}/{len(rows)} rows",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def write_sql(rows: list[tuple[int, str, str, str]]) -> None:
|
||||
lines = [
|
||||
"-- MoonWell static Russian localization for mod-playerbots.",
|
||||
"-- Generated after all upstream playerbots text migrations.",
|
||||
"-- Placeholders are validated by tools/generate-playerbots-ru-texts.py.",
|
||||
"SET NAMES utf8mb4;",
|
||||
"",
|
||||
"START TRANSACTION;",
|
||||
]
|
||||
for row_id, name, _source, target in rows:
|
||||
lines.append(
|
||||
"UPDATE `ai_playerbot_texts` "
|
||||
f"SET `text_loc8`={sql_quote(target)} "
|
||||
f"WHERE `id`={row_id} AND `name`={sql_quote(name)} AND LENGTH(`text_loc8`)=0;"
|
||||
)
|
||||
lines.extend(["COMMIT;", ""])
|
||||
OUTPUT.write_text("\n".join(lines), encoding="utf-8")
|
||||
print(f"Wrote {len(rows)} translations to {OUTPUT}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
rows = load_untranslated_rows()
|
||||
if not rows:
|
||||
raise SystemExit("No untranslated playerbot text rows found")
|
||||
write_sql(translate(rows))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user