#!/usr/bin/env python3 """Generate the final static ruRU update for untranslated playerbot texts. Run this after the playerbots database is fully updated. The generation-only dependencies are intentionally not part of the server runtime: pip install torch transformers sentencepiece sacremoses """ from __future__ import annotations import os import re import subprocess from collections import Counter from pathlib import Path import torch from transformers import AutoModelForSeq2SeqLM, AutoTokenizer ROOT = Path(__file__).resolve().parents[1] OUTPUT = ( ROOT / "modules/mod-playerbots/data/sql/playerbots/updates/" / "2026_07_30_00_ai_playerbot_russian_texts.sql" ) MODEL_NAME = os.environ.get( "PLAYERBOTS_RU_TRANSLATION_MODEL", "facebook/nllb-200-distilled-600M", ) PLACEHOLDER_RE = re.compile(r"%[A-Za-z_][A-Za-z0-9_]*|<[^>]+>") def load_untranslated_rows() -> list[tuple[int, str, str]]: sql = ( "SELECT id,HEX(name),HEX(text) " "FROM ai_playerbot_texts WHERE LENGTH(text_loc8)=0 ORDER BY id" ) command = [ "docker", "compose", "exec", "-T", "ac-database", "bash", "-lc", f'mysql -N -s -uroot -p"$MYSQL_ROOT_PASSWORD" acore_playerbots -e "{sql}" 2>/dev/null', ] output = subprocess.check_output(command, cwd=ROOT, text=True) rows: list[tuple[int, str, str]] = [] for line in output.splitlines(): parts = line.split("\t") if len(parts) != 3: continue row_id = int(parts[0]) name = bytes.fromhex(parts[1]).decode("utf-8") text = bytes.fromhex(parts[2]).decode("utf-8").replace("\ufffd", "'") rows.append((row_id, name, text)) return rows def sql_quote(value: str) -> str: return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'" def prepare_source(text: str) -> str: replacements = { "I’m": "I am", "I’ll": "I will", "I’ve": "I have", "won’t": "will not", "can’t": "cannot", "don’t": "do not", "Let’s": "Let us", "Just killed %victim_name": "I have just defeated %victim_name", "%victim_name was too easy": "Defeating %victim_name was too easy", "More %faction rep": "More reputation with %faction", "grinding %faction rep": "earning reputation with %faction", "farm %category": "collect %category", "farming %category": "collecting %category", "looting": "collecting loot", "hit me up": "message me", "Hit me up": "Message me", "hit level": "reached level", "Hit level": "Reached level", "turned it in": "handed it in", "over nothing": "rather than getting nothing", "Any takers": "Who wants", "a solid ": "a good ", "A solid ": "A good ", "great deal": "good price", "smack talk": "insults", } prepared = text for source, target in replacements.items(): prepared = prepared.replace(source, target) prepared = re.sub(r"\brep\b", "reputation", prepared, flags=re.IGNORECASE) prepared = re.sub(r"\bgrinding\b", "earning", prepared, flags=re.IGNORECASE) prepared = re.sub(r"\bgrind\b", "earn", prepared, flags=re.IGNORECASE) prepared = re.sub(r"\baggro\b", "attract enemies", prepared, flags=re.IGNORECASE) return prepared def translate(rows: list[tuple[int, str, str]]) -> list[tuple[int, str, str, str]]: is_nllb = "nllb" in MODEL_NAME.casefold() tokenizer_options = {"src_lang": "eng_Latn"} if is_nllb else {} tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, **tokenizer_options) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") dtype = torch.float16 if device.type == "cuda" else torch.float32 model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME, torch_dtype=dtype).to(device) batch_size = 48 if device.type == "cuda" else 12 result: list[tuple[int, str, str, str]] = [] def generate_texts(texts: list[str]) -> list[str]: encoded = tokenizer( texts, return_tensors="pt", padding=True, truncation=True, max_length=256, ) encoded = {key: value.to(device) for key, value in encoded.items()} generation_options = { "max_new_tokens": 256, "num_beams": 4, "early_stopping": True, } if is_nllb: generation_options["forced_bos_token_id"] = tokenizer.convert_tokens_to_ids("rus_Cyrl") generated = model.generate( **encoded, **generation_options, ) return tokenizer.batch_decode(generated, skip_special_tokens=True) def translate_with_segments(source: str) -> str: parts = re.split(f"({PLACEHOLDER_RE.pattern})", source) jobs: list[tuple[int, str, str, str]] = [] for part_index, part in enumerate(parts): if not part or PLACEHOLDER_RE.fullmatch(part): continue alpha_indexes = [index for index, char in enumerate(part) if char.isalpha()] if not alpha_indexes: continue first_alpha = alpha_indexes[0] last_alpha = alpha_indexes[-1] jobs.append( ( part_index, part[:first_alpha], prepare_source(part[first_alpha : last_alpha + 1]), part[last_alpha + 1 :], ) ) translated_parts = generate_texts([core for _index, _leading, core, _trailing in jobs]) for (part_index, leading, _core, trailing), target in zip(jobs, translated_parts): parts[part_index] = leading + target + trailing return "".join(parts) for start in range(0, len(rows), batch_size): batch = rows[start : start + batch_size] translated = generate_texts( [prepare_source(source) for _row_id, _name, source in batch] ) for (row_id, name, source), target in zip(batch, translated): source_placeholders = Counter(PLACEHOLDER_RE.findall(source)) target_placeholders = Counter(PLACEHOLDER_RE.findall(target)) if source_placeholders != target_placeholders: target = translate_with_segments(source) target_placeholders = Counter(PLACEHOLDER_RE.findall(target)) if source_placeholders != target_placeholders: raise RuntimeError( f"Placeholder mismatch for id={row_id}: " f"{source_placeholders} != {target_placeholders}" ) result.append((row_id, name, source, target)) print( f"Translated {min(start + batch_size, len(rows))}/{len(rows)} rows", flush=True, ) return result def write_sql(rows: list[tuple[int, str, str, str]]) -> None: lines = [ "-- MoonWell static Russian localization for mod-playerbots.", "-- Generated after all upstream playerbots text migrations.", "-- Placeholders are validated by tools/generate-playerbots-ru-texts.py.", "SET NAMES utf8mb4;", "", "START TRANSACTION;", ] for row_id, name, _source, target in rows: lines.append( "UPDATE `ai_playerbot_texts` " f"SET `text_loc8`={sql_quote(target)} " f"WHERE `id`={row_id} AND `name`={sql_quote(name)} AND LENGTH(`text_loc8`)=0;" ) lines.extend(["COMMIT;", ""]) OUTPUT.write_text("\n".join(lines), encoding="utf-8") print(f"Wrote {len(rows)} translations to {OUTPUT}") def main() -> None: rows = load_untranslated_rows() if not rows: raise SystemExit("No untranslated playerbot text rows found") write_sql(translate(rows)) if __name__ == "__main__": main()