From c2f72294d8cc986ddfe7311564bbc4edd03a3322 Mon Sep 17 00:00:00 2001 From: sindoring Date: Sat, 12 Sep 2026 16:30:07 +0400 Subject: [PATCH] =?UTF-8?q?=D0=BF=D0=BE=D0=B8=D1=81=D0=BA=20=D1=81=D1=82?= =?UTF-8?q?=D1=80=D0=BE=D0=BA=20=D0=BD=D0=B0=20=D1=84=D1=80=D0=B0=D0=BD?= =?UTF-8?q?=D1=86=D1=83=D0=B7=D1=81=D0=BA=D0=BE=D0=BC=20=D0=B4=D0=BB=D1=8F?= =?UTF-8?q?=20=D0=BF=D0=B5=D1=80=D0=B5=D0=B2=D0=BE=D0=B4=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../find_french_strings.cpython-312.pyc | Bin 0 -> 9178 bytes tools/find_french_strings.py | 156 ++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 tools/__pycache__/find_french_strings.cpython-312.pyc create mode 100644 tools/find_french_strings.py diff --git a/tools/__pycache__/find_french_strings.cpython-312.pyc b/tools/__pycache__/find_french_strings.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cb14a9671c0336507f8e32a19f1e716838984a3e GIT binary patch literal 9178 zcmbt3ZEzdMb$7S}aQGrX0{oCbO5le`K{iQ!P?jxPq$!aUsSi>XDBGf?Vj$d61Ox)q z-GLP0V*)2>1Gbt9irokrJ7Y9)TR3tXdXj1AHXXBrrq#4F9YCN8Vxw8Do^DaI8zf(ainPDm{|PC`rilvAp4l{BXGsi&xM3da-}Lou3rum&pL>U>*X z-z$80Ys!1`Fs<4Fz_^~#)9R28!)OZCzc1&wBh3u2ogp|JCqx!}Ts3#@Z}>b{0%;M9M*g_8BcyY`6tQul8NMZllPK;mi%7w2gwhUf0_JI z@}nQWk^JLiIQf<2*OLp$Nb+}+znA>|Brwl{?6s#f50-AuI%kS z>hc9xPH+X?LBHUdVcbv!Jgjhq;Q}7N;AVK9fjW3Ya0jNQ07SK)hZfGr2(-)32{bbu z^gxq*j|;FBg{yXXHX{` z$L+CSx)RzUo!2eUf>-bfQ-U9s6JV0PEmYYAoVawULwNPKnl4?sY~R-uva}Vt+S;JE zP3moP1J{ZLc{2jT@bzaTZ^jq=5qPA6B^NOPxC-9w+c;nFfEf0^Lbi9Wx*RVL(*#Y* zR zq56eN|N2Da+HhWSj$8G?lm2( zu%^fKRpGPiKnvzBS7D@}lA`~ur06s#BM7>_r0}4eFuu9S>LH7rR$PNsS$vKt%`B1@-12>;AO+3;f^$jUAn(Q-Wyouip;Akep#J94w+zYL;ETrP zwa8fl7>Y=EVanWJmR z<(5JLQNHMX0k_M?#r4nX0x-#oqAARlV^|uq^kBWw*0XDgZA%t{>{O^JNDPd zGSrLfgkq;EMb><(!YE_TXhuZfUkOE-*!6FCkgA2h%4 zgXTs0`}U>lA9SS1KDHCccNZ@l_&^Rwve9V(?qGrxSntH<0rHmP!6+)_7tEI9N?Ijf zKwFNhX-XcKv`NyCPyPqW2CmsqT;exyLr8In-N3cb_)*jb@tM!%W&l*qxO_o|Lv+bv za{CyU-wA#iu5}-K7DoEupL-T65R<+#+88|^IUOFz=}fndEgXwkV!@1VPgd8S*0m?- z#et0OXjXSTtvkNlb-y~J8%>d;c_zUE#ez&&V$#3DU3v%It7S$a8|XaF7mToU!}ViP z{POqn_p4V~T(1c5NbRH|8}3%Ob#VtmgKd3B(>TNh1#x*1wVO(D1(Gki7;poGEDDKE zFz`!Y)AP!C6<>5QSMhl@?778z$cvssn~J;-O-fOIwV?J|H=)2MUoZE`G=l?D&QpRK zqLD8F1sXx)-7VvkT0te_1KuRyY57KZRM2{xWw_LWG8VWZx86OaQ5lk{p4ahha-X0R zijfw?CeI3lpcHfy#Jpb63-}Dqwt|6daW;ep&Qv!+$~)o^v3TwBx}{domoR8*p1SEJ zv72FtZ&aU4qv6+(hDLJjl|YHX`&5Yn8KSi^CG!IHi`ZgbYO0`)2u4K8(#91*7-55{3-XmJ(G0 z23p+$HV7_xo&YN}0r$3?B@l06v`$b;wMnXVP*;76Dy??2mBciVDM5cFx;@x@rL+%l z>4|+Rz1=cRrB*P3EEHFZ^S9P_*tr_f@~!eX{Q!ya&&w-$e={D{lupI>%4-UhUgYRE z71@&MT1n!CU=;XzEbrSS{|0=WuiA#Q5;yr1^0GpeymyhyI=ap~Adi>YCorFsMDI3| zFiVnnd6Ohev>N$N@HrLmqT0$@cgQpWj%^#9oiYxK-#jRox0J5#*OKf-knEs*?%zD9=Z98e80VUghf|7M4S}LL(z~`l;Tttg+YTXax0>` z5Oa?8pidShli6CR?8=a$vvUJ)hI~Xjfcr#zAA1sw4nY;t6%t{cko9{a5J%bou^GPJ z3P3&(M*$WDF zK*|rNF?9NG;&*UxTRJc${7;El9B4s0GQvv7R)~s@{p?H7!65}Myn8n^&tGNy%#CTb zCseyhQRIOrnu8t)|Bn4Kj6JC7h&M;M*nvNoN!N7zx0KQPim?(8QRk(+`cYd*JqB5L z9{|(^DSZymjyz;5B+ntKFFoKm7*ZUy*;_?|V|XdgCF)ql2a=y*obU^P4Yp z%eZKXO^~5Oi8`cMoLB`5qwHP3l*1c<^dBp#M;&=U!}x=cB!+yTXy_lkaQw`G^Ti9J zV*~vxdKIw9@>sZg;q~tr9ywVY6)BXVKom+3Oi_824Fspz!w9_}p%X5eX3_J5MgCbN zA)U>Os?iH$=Pp1xF>fKFa_SoGVMQ_kX+g+Ku&^Fb&hRt?DQ!p#8ZL|szj$H5DNjR- zFtn(Yl#}g}7WP7JU932NVPtIhtc=M*yoia)Y547fx7UayGJGI`Vf^`A2qp6m5|t8* zbf|FxC$2!&l0+XO1<6P_o}Uv96Yv`eEkCG{c}oL8y^D8GvdHVRub>SnDT^}=6jpjT zXThsE1FRDqn+T~+^qY!EB9j(%Wy&s=lv*^)Kb@41Q_~FVl%^@&=;s;^`CLfLY*F%= z8WO1~dke5oSi(I9cEHR3>5g#!YODP_Ot}B&riM3+>sXVr2{M}I+N`-bZEk+IKV!CM z%?H!wgBf#Igj!P&n(CaT?#{^Vk;KX9NXF8Ywe+Sfy%|eiWZ*LjYkDeic(F0l*tL#X zHBD)KL*#VKvRbz%`sK(FWL<8(w(wfkXiFPy8Dmp?;(_t$oT=`P@wV|@!-}cnqv~8kOSYjs-2fSgOv9l^n7Xn(IzHK9=ZuSi3*ByDec|q*J@PV?@rnD^B0-&DGo9IeqtZqHeK!S#!TDwfEdg z{fnq`=rpC0X5mHm} zFQ-h;Ji_qG0X%26X3ec>b8D*YM8@0?Ib_4GbqX^z-fQ^LvHNu?$IJiJ{YtKOceb`I zUE7u*6HKPID@sC7+?t?MHHV@^&Ri3t-{?hLJ$?Ihye`qbWLV}?wlgc1v#4|E_E606 z&9je=07KV|AfL4%98+q(KK%M{x~eBUyh8S_lG?AIef?~ftWA@(aWcWD$l4XMYgJ>s zWnHjF9q}WJ{Sj+M^V}LnD2MT-mvU6aE&YN%RsGBgbtHG-FdRlRjHmVOk)i0dcvr%L z(${Ae&cuddN8{%cy7>I!@kM@dV(Iu2zcjIYJhkKGL;cXF1e*0&iPl`xVuz1}Pv_bX zz*@=+c$%_DsOVEMK0cPC%vowzn%Wg7<8(|&oX=4Ev(%9^b!6$;rG3kN%h#4Kr=E2@ zq{h|=H1+42nyki})>vaF5^YOeDGgBm!WyPlRxA$|2|vC!JNG4;5@U(M#iqpz zKOFyHd|8t@d@9v-`u@R`Y2?2Mg`wh8H7xX42^+6fVFwNYQ>Ejnlo!j7_p#m-9nR<* zK$Ekk=Cr9f)zbCQbU0_J&sy5jmbQfMp{46%{qC%OUs}H}(Y08W(RXF_-D!RI(v2nW zef<7Y_Zw2rzx+`D%9;VK{aFR3t;=fa)0+C2BXMM@Kc%V9X!_RRcZiDRlITvYP^VXS z?@6^jmu>A$xAtaQ`!c(aKf(x2#lmQ$KWZF^5-RB?qelv10wue+B zsGPFtm!FL)u*&9lPo|7dBh9TA^%hVaP4&j(ejF*|r;?g)JT`&oK1GVN@wf?2C2??r zgBKC@k?s6p#r>V#4ju9LPxo~=>WF{b(YMF3hj?(<=%ADtHI8bX(owC>)F~ZJ>dYRc zqeCrHPA77tPA3F);J*1F>knyt5YoMhek#9gS4qAf?q!~p?iCgVtn6)69fitHvS$$d zjbbl?Aml(8A~_Q9GxlAg+KH|Qr&H8AofARWgK;`p6vDCHsM>{aAwu@~n0%@}f48Aa zXwv7oB5jEXMS&zN0-0eh;G1FCi)gY9RVa(i&Y&uUsuonCoSvwL8)_P+vCp8c9#o+m zG5Z{zRf4bd-lKq(5Gr1)35c_5~q3eZ6x)5E?E11QnRX`u#z z;rPE`XZ{Uq`WdGAXRPw4*zTWVbtrAGj|_eFOIdY8THTOQ@6M{*((1MZo>uP*lR1qc zs!3~hr0VylHT%P=oXHZMOPg$|=AN{vCv3=7R>w5y%Ena7$#mt(a7C`-Tv+!=K~yTj z25?cj>PXYAeGB_i)#DlMt6_DHs){IXX&1C`u_?ROC`_qFj&xn8(kQ~pHL!x65%yMS zA(Yzr%#z|G!-s~9aWJgUnRZ4MceJ;)DQjoOv>&!DbsWr?4uuWtDji<6UT?sULZ~C#Ezek)Nc>25g55s;?@5Gk)l@8 literal 0 HcmV?d00001 diff --git a/tools/find_french_strings.py b/tools/find_french_strings.py new file mode 100644 index 0000000..c6a36c3 --- /dev/null +++ b/tools/find_french_strings.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 + + +"""Build a reviewable TSV inventory of likely French first-party strings.""" + +from __future__ import annotations + +import re +import subprocess +from collections import Counter +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +OUTPUT = ROOT / "reports" / "french_strings.tsv" +UNIQUE_OUTPUT = ROOT / "reports" / "french_strings_unique.tsv" +RUNTIME_OUTPUT = ROOT / "reports" / "french_runtime_strings_unique.tsv" +FIRST_PARTY = ("src/", "sql/", "sylvania/", "chantiers/", "docker/", "contrib/") +TEXT_SUFFIXES = { + ".c", ".cc", ".cpp", ".cxx", ".h", ".hpp", ".inl", + ".sql", ".md", ".txt", ".conf", ".dist", ".sh", ".bat", +} + +ACCENTS = re.compile(r"[àâçéèêëîïôùûüÿœæÀÂÇÉÈÊËÎÏÔÙÛÜŸŒÆ]") +WORDS = re.compile( + r"\b(?:alors|aucun|avec|avoir|besoin|cette|ceux|comme|comment|contre|" + r"dans|depuis|désormais|doit|donc|encore|entre|êtes|faire|faut|" + r"jusqu|leur|leurs|mais|merci|notre|pour|pourquoi|quand|sans|sera|" + r"sont|tout|tous|très|une|votre|vous|voici)\b", + re.IGNORECASE, +) +CONTRACTIONS = re.compile(r"\b(?:c|d|j|l|m|n|qu|s|t)'", re.IGNORECASE) +DOUBLE_QUOTED = re.compile(r'"(?:\\.|[^"\\])*"') +SINGLE_QUOTED = re.compile(r"'(?:\\.|''|[^'\\])*'") + + +def tracked_files() -> list[Path]: + output = subprocess.check_output( + ["git", "ls-files", "-z"], cwd=ROOT + ).decode("utf-8", "surrogateescape") + result = [] + for name in output.split("\0"): + if not name or not name.startswith(FIRST_PARTY): + continue + path = ROOT / name + if path.suffix.lower() in TEXT_SUFFIXES: + result.append(path) + return result + + +def likely_french(text: str, path: Path) -> bool: + if "�" in text or len(text.strip()) < 2: + return False + accents = len(ACCENTS.findall(text)) + words = len(WORDS.findall(text)) + contractions = len(CONTRACTIONS.findall(text)) + french_locale_file = bool( + re.search(r"(?:frfr|_fr(?:\.|_)|french)", path.as_posix(), re.IGNORECASE) + ) + if accents: + return True + if words >= 2 or (words >= 1 and contractions >= 1): + return True + return french_locale_file and (" " in text or len(text) >= 8) + + +def clean_literal(value: str) -> str: + value = value[1:-1] + return value.replace("\t", "\\t").replace("\r", "\\r").replace("\n", "\\n") + + +def main() -> None: + rows: list[tuple[str, int, str, str]] = [] + seen: set[tuple[str, int, str]] = set() + + for path in tracked_files(): + relative = path.relative_to(ROOT).as_posix() + try: + lines = path.read_text(encoding="utf-8").splitlines() + except (UnicodeDecodeError, OSError): + continue + + for line_number, line in enumerate(lines, 1): + patterns = [DOUBLE_QUOTED] + if path.suffix.lower() == ".sql": + patterns.append(SINGLE_QUOTED) + + found_literal = False + for pattern in patterns: + for match in pattern.finditer(line): + text = clean_literal(match.group(0)).strip() + if likely_french(text, path): + key = (relative, line_number, text) + if key not in seen: + rows.append((relative, line_number, "literal", text)) + seen.add(key) + found_literal = True + + # Documentation and comments may not use quoted literals. + stripped = line.strip() + is_comment_or_doc = ( + path.suffix.lower() in {".md", ".txt"} + or stripped.startswith(("//", "#", "--", "/*", "*")) + ) + if is_comment_or_doc and not found_literal and likely_french(stripped, path): + text = stripped.replace("\t", "\\t") + key = (relative, line_number, text) + if key not in seen: + rows.append((relative, line_number, "comment/doc", text)) + seen.add(key) + + rows.sort(key=lambda row: (row[0], row[1], row[2], row[3])) + OUTPUT.parent.mkdir(parents=True, exist_ok=True) + with OUTPUT.open("w", encoding="utf-8", newline="\n") as report: + report.write("file\tline\tkind\tfrench_text\n") + for relative, line_number, kind, text in rows: + report.write(f"{relative}\t{line_number}\t{kind}\t{text}\n") + + occurrences: dict[tuple[str, str], list[str]] = {} + for relative, line_number, kind, text in rows: + occurrences.setdefault((kind, text), []).append(f"{relative}:{line_number}") + with UNIQUE_OUTPUT.open("w", encoding="utf-8", newline="\n") as report: + report.write("kind\toccurrences\tfrench_text\tlocations\n") + for (kind, text), locations in sorted( + occurrences.items(), key=lambda item: (item[0][0], item[0][1]) + ): + report.write( + f"{kind}\t{len(locations)}\t{text}\t{' | '.join(locations)}\n" + ) + + runtime_occurrences: dict[str, list[str]] = {} + for relative, line_number, kind, text in rows: + if ( + kind == "literal" + and relative.startswith(("src/", "sql/")) + and not relative.startswith("sql/test/") + ): + runtime_occurrences.setdefault(text, []).append(f"{relative}:{line_number}") + with RUNTIME_OUTPUT.open("w", encoding="utf-8", newline="\n") as report: + report.write("occurrences\tfrench_text\trussian_text\tlocations\n") + for text, locations in sorted(runtime_occurrences.items()): + report.write( + f"{len(locations)}\t{text}\t\t{' | '.join(locations)}\n" + ) + + per_kind = Counter(row[2] for row in rows) + print(f"Report: {OUTPUT.relative_to(ROOT)}") + print(f"Total: {len(rows)}") + print(f"Unique: {len(occurrences)}") + print(f"Unique runtime literals: {len(runtime_occurrences)}") + for kind, count in sorted(per_kind.items()): + print(f"{kind}: {count}") + + +if __name__ == "__main__": + main()