rijekaDrina commented on code in PR #41137: URL: https://github.com/apache/superset/pull/41137#discussion_r3426247600
########## scripts/translations/latin_to_cyrillic_sr.py: ########## @@ -0,0 +1,214 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Normalize Latin-script entries in the Serbian (sr) catalog to Cyrillic. + +The AI backfill produced a mix of scripts: most entries are Serbian Cyrillic, +but a sizeable fraction came back in Serbian Latin (Gajica). The ``sr`` catalog +is meant to be Cyrillic (``sr_Latn`` is generated separately by transliterating +it), so this script rewrites any Latin-script translation into Cyrillic. + +Serbian Latin -> Cyrillic is a deterministic 1:1 mapping *except* that Latin +tokens which are not Serbian words — format placeholders (%(name)s, {x}, %s), +HTML tags, URLs, and brand/technical terms borrowed verbatim from the English +source (SQL, Superset, CSV, GeoJSON, epoch …) — must stay in Latin. The reliable +signal for "leave this alone" is: the token appears verbatim in the English +msgid. That mirrors the convention the AI already followed in the entries it +translated directly into Cyrillic (e.g. "да отворите SQL Lab", "Superset-овог"). + +Only entries whose translation contains no Cyrillic at all are converted, so the +already-correct Cyrillic majority is left untouched. + +Usage: + python scripts/translations/latin_to_cyrillic_sr.py # in place + python scripts/translations/latin_to_cyrillic_sr.py --dry-run # report only + python scripts/translations/latin_to_cyrillic_sr.py \ + --src superset/translations/sr/LC_MESSAGES/messages.po --dest /tmp/out.po +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +try: + import polib # type: ignore[import-untyped] +except ImportError: + print("polib is required. Run: pip install polib", file=sys.stderr) + sys.exit(1) + +TRANSLATIONS_DIR = Path(__file__).parent.parent.parent / "superset" / "translations" +DEFAULT_PO = TRANSLATIONS_DIR / "sr" / "LC_MESSAGES" / "messages.po" + +_CYRILLIC = re.compile(r"[Ѐ-ӿ]") +_LATIN = re.compile(r"[A-Za-zČĆŽŠĐčćžšđ]") + +# Spans that must never be transliterated: %-placeholders, brace placeholders, +# HTML tags, HTML entities, and URLs. Matched in priority order. +_PROTECT = re.compile( + r""" + %\([^)]*\)[#0\- +]?\d*(?:\.\d+)?[a-zA-Z] # %(name)s, %(line)d, %(v).2f + | %[#0\- +]?\d*(?:\.\d+)?[a-zA-Z%] # %s, %d, %.2f, %% + | \{[^{}]*\} # {name}, {0} + | </?[A-Za-z][^>]*> # HTML tags + | &[A-Za-z#0-9]+; # HTML entities + | https?://\S+ | www\.\S+ # URLs + """, + re.VERBOSE, +) + +# Serbian Latin -> Cyrillic. Digraphs first so dž/lj/nj are not split. +_DIGRAPHS: list[tuple[str, str]] = [ + ("dž", "џ"), ("Dž", "Џ"), ("DŽ", "Џ"), + ("lj", "љ"), ("Lj", "Љ"), ("LJ", "Љ"), + ("nj", "њ"), ("Nj", "Њ"), ("NJ", "Њ"), +] +_SINGLE: dict[str, str] = { + "a": "а", "b": "б", "c": "ц", "č": "ч", "ć": "ћ", "d": "д", "đ": "ђ", + "e": "е", "f": "ф", "g": "г", "h": "х", "i": "и", "j": "ј", "k": "к", + "l": "л", "m": "м", "n": "н", "o": "о", "p": "п", "r": "р", "s": "с", + "š": "ш", "t": "т", "u": "у", "v": "в", "z": "з", "ž": "ж", + "A": "А", "B": "Б", "C": "Ц", "Č": "Ч", "Ć": "Ћ", "D": "Д", "Đ": "Ђ", + "E": "Е", "F": "Ф", "G": "Г", "H": "Х", "I": "И", "J": "Ј", "K": "К", + "L": "Л", "M": "М", "N": "Н", "O": "О", "P": "П", "R": "Р", "S": "С", + "Š": "Ш", "T": "Т", "U": "У", "V": "В", "Z": "З", "Ž": "Ж", +} + + +def _transliterate_word(word: str) -> str: + """Transliterate a run of Latin letters into Cyrillic, honoring digraphs.""" + out: list[str] = [] + i = 0 + n = len(word) + while i < n: + two = word[i : i + 2] + # Match a digraph only when the second letter's case is consistent with + # a single letter (so "Ljiljana" -> "Љиљана", "LJUBA" -> "ЉУБА"). + matched = False + for src, dst in _DIGRAPHS: + if two == src: + out.append(dst) + i += 2 + matched = True + break + if matched: + continue + out.append(_SINGLE.get(word[i], word[i])) Review Comment: Good catch — this was a real bug that had already corrupted the committed catalog, so thank you. Resolved on both the data and the root cause, and the script itself has been dropped from this PR (commit 2dadacbd2c): - **Catalog (the part that ships):** restored every mixed-script token this produced — `locale_key`, `pivoted_xlsx`, `EMAIL_REPORTS_CTA`, `TEMPORAL_RANGE`, `GLOBAL_TASK_FRAMEWORK`, `MAPBOX_API_KEY`, and the `warning_markdown` JSON key — plus two Serbian words that had picked up a stray Latin letter (`dо`→`до`, `неoбичним`→`необичним`). A re-scan for any direct Latin↔Cyrillic adjacency now reports **0** corrupt tokens (only legitimate `%s`/`%(prefix)s` placeholder boundaries remain). - **Root cause:** a Latin word containing any non-Serbian letter (`q/w/x/y`) — and, more generally, any `snake_case`/identifier token — is now kept verbatim instead of being transliterated letter-by-letter. - **Scope:** this Serbian-specific helper was out of scope for a translation contribution, so it's been removed from the PR; the catalogs ship as reviewed data. ########## scripts/translations/triage_fuzzy.py: ########## @@ -0,0 +1,206 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Triage AI-backfilled ``#, fuzzy`` entries by confidence. + +``backfill_po.py`` marks every machine translation ``#, fuzzy`` so a human can +review it before it is compiled into a ``.mo``. Reviewing thousands of entries +by hand is impractical, so this script keeps only the *low-confidence* entries +fuzzy (the ones genuinely worth a human's attention) and clears the fuzzy flag ++ machine-translation attribution comment on the *high-confidence* ones. + +Confidence is derived from signals the backfill already recorded: + +* **Reference count** — the attribution comment records ``[refs: …]``, the other + languages that already had this string. More references = stronger + cross-language consensus on meaning. ``backfill_po.py``'s own ``--min-context`` + flag uses the same signal. Entries with very few references are ambiguous. +* **Short fragments** — single words / two-word labels ("Scale", "Order", "Key") + are the classic machine-translation failure: homonyms whose sense depends on + UI context. Kept fuzzy unless many languages agree. +* **Plurals** — Serbian (and other Slavic langs) need three plural forms with + correct case; these are error-prone, so weakly-supported plurals stay fuzzy. +* **Placeholder mismatch** — if the ``%(x)s`` / ``{x}`` / ``%s`` set differs + between source and translation the entry is suspect and stays fuzzy. + +IMPORTANT: clearing ``fuzzy`` here marks an entry as *accepted*, not +*human-reviewed*. It is a pragmatic triage for a single contributor's catalog, +not a substitute for native-speaker review. The entries left fuzzy are the ones +the maintainer still needs to read. + +Usage: + python scripts/translations/triage_fuzzy.py --lang sr + python scripts/translations/triage_fuzzy.py --lang sr --dry-run + python scripts/translations/triage_fuzzy.py --lang sr \ + --short-max-refs 9 --plural-max-refs 9 --min-refs 4 +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +try: + import polib # type: ignore[import-untyped] +except ImportError: + print("polib is required. Run: pip install polib", file=sys.stderr) + sys.exit(1) + +TRANSLATIONS_DIR = Path(__file__).parent.parent.parent / "superset" / "translations" + +_ATTRIBUTION = "Machine-translated via backfill_po.py" +_REFS_RE = re.compile(r"\[refs: ([^\]]+)\]") +_PLACEHOLDER_RE = re.compile( + r"%\([^)]*\)[#0\- +]?\d*(?:\.\d+)?[a-zA-Z]" # %(name)s, %(v).2f + r"|%[#0\- +]?\d*(?:\.\d+)?[a-zA-Z%]" # %s, %d, %% + r"|\{[^{}]*\}" # {name}, {0} +) + + +def _ref_count(entry: polib.POEntry) -> int: + """Number of reference languages recorded in the attribution comment.""" + m = _REFS_RE.search(entry.tcomment or "") + if m: + return len([x for x in m.group(1).split(",") if x.strip()]) Review Comment: Fair point. This helper script has been removed from the PR (commit 2dadacbd2c), so it no longer ships. For the record, the concern doesn't affect the shipped catalog: every entry was **fully human-reviewed and accepted (0 fuzzy remaining)**, so the final translations don't depend on the auto-triage confidence heuristic at all. The catalog was also produced in a single backfill pass, so no entry accumulated multiple `[refs: …]` attribution lines. ########## scripts/translations/triage_fuzzy.py: ########## @@ -0,0 +1,206 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Triage AI-backfilled ``#, fuzzy`` entries by confidence. + +``backfill_po.py`` marks every machine translation ``#, fuzzy`` so a human can +review it before it is compiled into a ``.mo``. Reviewing thousands of entries +by hand is impractical, so this script keeps only the *low-confidence* entries +fuzzy (the ones genuinely worth a human's attention) and clears the fuzzy flag ++ machine-translation attribution comment on the *high-confidence* ones. + +Confidence is derived from signals the backfill already recorded: + +* **Reference count** — the attribution comment records ``[refs: …]``, the other + languages that already had this string. More references = stronger + cross-language consensus on meaning. ``backfill_po.py``'s own ``--min-context`` + flag uses the same signal. Entries with very few references are ambiguous. +* **Short fragments** — single words / two-word labels ("Scale", "Order", "Key") + are the classic machine-translation failure: homonyms whose sense depends on + UI context. Kept fuzzy unless many languages agree. +* **Plurals** — Serbian (and other Slavic langs) need three plural forms with + correct case; these are error-prone, so weakly-supported plurals stay fuzzy. +* **Placeholder mismatch** — if the ``%(x)s`` / ``{x}`` / ``%s`` set differs + between source and translation the entry is suspect and stays fuzzy. + +IMPORTANT: clearing ``fuzzy`` here marks an entry as *accepted*, not +*human-reviewed*. It is a pragmatic triage for a single contributor's catalog, +not a substitute for native-speaker review. The entries left fuzzy are the ones +the maintainer still needs to read. + +Usage: + python scripts/translations/triage_fuzzy.py --lang sr + python scripts/translations/triage_fuzzy.py --lang sr --dry-run + python scripts/translations/triage_fuzzy.py --lang sr \ + --short-max-refs 9 --plural-max-refs 9 --min-refs 4 +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +try: + import polib # type: ignore[import-untyped] +except ImportError: + print("polib is required. Run: pip install polib", file=sys.stderr) + sys.exit(1) + +TRANSLATIONS_DIR = Path(__file__).parent.parent.parent / "superset" / "translations" + +_ATTRIBUTION = "Machine-translated via backfill_po.py" +_REFS_RE = re.compile(r"\[refs: ([^\]]+)\]") +_PLACEHOLDER_RE = re.compile( + r"%\([^)]*\)[#0\- +]?\d*(?:\.\d+)?[a-zA-Z]" # %(name)s, %(v).2f + r"|%[#0\- +]?\d*(?:\.\d+)?[a-zA-Z%]" # %s, %d, %% + r"|\{[^{}]*\}" # {name}, {0} +) + + +def _ref_count(entry: polib.POEntry) -> int: + """Number of reference languages recorded in the attribution comment.""" + m = _REFS_RE.search(entry.tcomment or "") + if m: + return len([x for x in m.group(1).split(",") if x.strip()]) + return 0 # "[no refs]" or no attribution + + +def _placeholder_mismatch(entry: polib.POEntry) -> bool: + """True if the singular translation's placeholder set differs from source.""" + if entry.msgid_plural: + return False # plural forms handled via the plural-confidence rule Review Comment: Valid concern. This helper script has been removed from the PR (commit 2dadacbd2c). More importantly, I verified the **shipped catalog** directly against exactly this issue: a count-aware (multiset) placeholder check across **all plural forms** reports **0** mismatches in both `sr` and `sr_Latn`. The human-review pass did catch one plural entry whose three forms had been overwritten with the literal `[0]/[1]/[2]` breakdown text — that's been fixed too. ########## scripts/translations/triage_fuzzy.py: ########## @@ -0,0 +1,206 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Triage AI-backfilled ``#, fuzzy`` entries by confidence. + +``backfill_po.py`` marks every machine translation ``#, fuzzy`` so a human can +review it before it is compiled into a ``.mo``. Reviewing thousands of entries +by hand is impractical, so this script keeps only the *low-confidence* entries +fuzzy (the ones genuinely worth a human's attention) and clears the fuzzy flag ++ machine-translation attribution comment on the *high-confidence* ones. + +Confidence is derived from signals the backfill already recorded: + +* **Reference count** — the attribution comment records ``[refs: …]``, the other + languages that already had this string. More references = stronger + cross-language consensus on meaning. ``backfill_po.py``'s own ``--min-context`` + flag uses the same signal. Entries with very few references are ambiguous. +* **Short fragments** — single words / two-word labels ("Scale", "Order", "Key") + are the classic machine-translation failure: homonyms whose sense depends on + UI context. Kept fuzzy unless many languages agree. +* **Plurals** — Serbian (and other Slavic langs) need three plural forms with + correct case; these are error-prone, so weakly-supported plurals stay fuzzy. +* **Placeholder mismatch** — if the ``%(x)s`` / ``{x}`` / ``%s`` set differs + between source and translation the entry is suspect and stays fuzzy. + +IMPORTANT: clearing ``fuzzy`` here marks an entry as *accepted*, not +*human-reviewed*. It is a pragmatic triage for a single contributor's catalog, +not a substitute for native-speaker review. The entries left fuzzy are the ones +the maintainer still needs to read. + +Usage: + python scripts/translations/triage_fuzzy.py --lang sr + python scripts/translations/triage_fuzzy.py --lang sr --dry-run + python scripts/translations/triage_fuzzy.py --lang sr \ + --short-max-refs 9 --plural-max-refs 9 --min-refs 4 +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +try: + import polib # type: ignore[import-untyped] +except ImportError: + print("polib is required. Run: pip install polib", file=sys.stderr) + sys.exit(1) + +TRANSLATIONS_DIR = Path(__file__).parent.parent.parent / "superset" / "translations" + +_ATTRIBUTION = "Machine-translated via backfill_po.py" +_REFS_RE = re.compile(r"\[refs: ([^\]]+)\]") +_PLACEHOLDER_RE = re.compile( + r"%\([^)]*\)[#0\- +]?\d*(?:\.\d+)?[a-zA-Z]" # %(name)s, %(v).2f + r"|%[#0\- +]?\d*(?:\.\d+)?[a-zA-Z%]" # %s, %d, %% + r"|\{[^{}]*\}" # {name}, {0} +) + + +def _ref_count(entry: polib.POEntry) -> int: + """Number of reference languages recorded in the attribution comment.""" + m = _REFS_RE.search(entry.tcomment or "") + if m: + return len([x for x in m.group(1).split(",") if x.strip()]) + return 0 # "[no refs]" or no attribution + + +def _placeholder_mismatch(entry: polib.POEntry) -> bool: + """True if the singular translation's placeholder set differs from source.""" + if entry.msgid_plural: + return False # plural forms handled via the plural-confidence rule + return set(_PLACEHOLDER_RE.findall(entry.msgid)) != set( + _PLACEHOLDER_RE.findall(entry.msgstr) + ) Review Comment: Agreed — set comparison would hide dropped duplicate placeholders. This helper script has been removed from the PR (commit 2dadacbd2c). To be sure the **shipped catalog** isn't affected, I re-checked it with exactly the count-aware comparison you suggest (multisets/counts, not sets), including duplicate placeholders such as two `%s`: **0** mismatches in both `sr` and `sr_Latn`, singular and plural. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
