This is an automated email from the ASF dual-hosted git repository.
jimjag pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/openoffice-devtools.git
The following commit(s) were added to refs/heads/main by this push:
new 924c90c translation-scripts: add sdf_integrate.py for merging PO
translations
924c90c is described below
commit 924c90cff85b2706336b7f2d88ae2742c159e300
Author: Jim Jagielski <[email protected]>
AuthorDate: Wed Nov 11 16:23:44 2026 -0500
translation-scripts: add sdf_integrate.py for merging PO translations
Replacing localize.sdf with po2oo output loses tree translations that are
fuzzy or empty in Pootle, so this merges changed entries and gates on
gsicheck.
---
translation-scripts/integrate/README.md | 54 ++++++
translation-scripts/integrate/sdf_integrate.py | 257 +++++++++++++++++++++++++
2 files changed, 311 insertions(+)
diff --git a/translation-scripts/integrate/README.md
b/translation-scripts/integrate/README.md
new file mode 100644
index 0000000..ed85cf8
--- /dev/null
+++ b/translation-scripts/integrate/README.md
@@ -0,0 +1,54 @@
+# Integrating Pootle translations into the source tree
+
+`sdf_integrate.py` brings translations from the PO files in
+[apache/openoffice-translation](https://github.com/apache/openoffice-translation)
+into `extras/l10n/source/<lang>/localize.sdf`.
+
+It merges instead of replacing the file. `po2oo` writes English for every entry
+that is fuzzy or empty in Pootle, and many of those are translated in the tree,
+so a whole-file replacement loses them (15,441 entries for `am` in 2026).
+
+Needs Python 3.7+, translate-toolkit (`po2oo`) and `gsicheck` from a built
+OpenOffice tree. No other dependencies.
+
+## Steps, for one language
+
+1. Convert the PO files with the English SDF they were made from:
+
+ po2oo --template=en-US.sdf --skipsource --keeptimestamp \
+ --language=de PO-Files/de de.po2oo.sdf
+
+ `po2oo` writes the `--language` value into every line, so use the tree's
+ code (`pt-BR`) here and for `--lang` below. Only the PO directory keeps
+ Pootle's form (`PO-Files/pt_BR`).
+
+2. Optional: if a newer snapshot may have rolled translations back, convert
+ the trusted snapshot, the newer one and some older ones, then combine.
+ A newer value is taken only if it is not blank and does not match text
+ from any of the `--history` files.
+
+ sdf_integrate.py combine --lang de --template en-US.sdf \
+ --base de.2022.sdf --newer de.head.sdf \
+ --history de.2019.sdf de.2021.sdf --out de.po2oo.sdf
+
+3. Merge and gate:
+
+ sdf_integrate.py gate --lang de \
+ --tree extras/l10n/source/de/localize.sdf \
+ --new de.po2oo.sdf --template en-US.sdf --out de.merged.sdf \
+ --gsicheck main/solver/<ver>/<platform>/bin/gsicheck \
+ --libdir main/solver/<ver>/<platform>/lib \
+ --skip-out de.left-out.json
+
+ Only entries whose translation changed are updated. Every other line, its
+ line ending and its timestamp stay as they are. Keys that are new and
+ translated are added after the last line of the same source file.
+
+ Entries whose new text adds a `gsicheck` error that the tree file does not
+ already have keep their tree text, and are listed as "left out". Report
+ those to the language team. The command exits 1 if any new error is left
+ that it cannot tie to an entry.
+
+4. Copy `de.merged.sdf` over the tree file and commit it, one commit per
+ language. `merge --skip de.left-out.json` gives the same result without
+ running `gsicheck` again.
diff --git a/translation-scripts/integrate/sdf_integrate.py
b/translation-scripts/integrate/sdf_integrate.py
new file mode 100755
index 0000000..fbd2a96
--- /dev/null
+++ b/translation-scripts/integrate/sdf_integrate.py
@@ -0,0 +1,257 @@
+#!/usr/bin/env python3
+################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+################################################################
+"""Merge po2oo output into an existing localize.sdf.
+
+ combine build one po2oo SDF from a base and a newer snapshot, ignoring
+ newer values that are blank or revert to text from older snapshots
+ merge update only the entries whose translation changed
+ gate merge, then run gsicheck and keep the tree text for every entry
+ that adds a gsicheck error the tree file does not already have
+
+See README.md for the full workflow.
+"""
+
+import argparse
+import collections
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+
+TEXT_COLUMNS = range(10, 14) # text, helptext, quickhelptext, title
+
+
+def key(cols):
+ # project, file, type, gid, lid, helpid, platform
+ return (cols[0], cols[1], cols[3], cols[4], cols[5], cols[6], cols[7])
+
+
+def load(path, lang):
+ entries = {}
+ with open(path, encoding="utf-8") as f:
+ for line in f:
+ cols = line.rstrip("\r\n").split("\t")
+ if len(cols) == 15 and cols[9] == lang:
+ entries[key(cols)] = cols
+ return entries
+
+
+def is_translation(value, english):
+ return bool(value) and value != english
+
+
+def combine(lang, template, base_path, newer_path, history_paths):
+ en = load(template, "en-US")
+ base, newer = load(base_path, lang), load(newer_path, lang)
+ history = collections.defaultdict(set)
+ for path in history_paths:
+ for k, cols in load(path, lang).items():
+ for i in TEXT_COLUMNS:
+ history[(k, i)].add(cols[i])
+
+ stats = collections.Counter()
+ out = []
+ for k, a in base.items():
+ row = list(a)
+ b = newer.get(k)
+ if b:
+ for i in TEXT_COLUMNS:
+ if b[i] == a[i]:
+ continue
+ english = en[k][i] if k in en else None
+ if not is_translation(b[i], english):
+ stats["newer blank or English, kept base"] += 1
+ elif b[i] in history[(k, i)]:
+ stats["newer reverts to older text, kept base"] += 1
+ else:
+ row[i] = b[i]
+ stats["newer value taken"] += 1
+ out.append("\t".join(row) + "\n")
+ for k, b in newer.items():
+ if k not in base:
+ out.append("\t".join(b) + "\n")
+ stats["key only in newer"] += 1
+ return "".join(out), stats
+
+
+def merge(lang, tree_path, new_path, template, skip=()):
+ """Return (text, changed keys, added keys)."""
+ new = load(new_path, lang)
+ if not new:
+ sys.exit("no %s lines in %s; po2oo --language must use the tree's code
(pt-BR, not pt_BR)"
+ % (lang, new_path))
+ en = load(template, "en-US")
+ skip = set(skip)
+ with open(tree_path, encoding="utf-8", newline="") as f:
+ lines = f.read().splitlines(keepends=True)
+
+ seen, changed, last_of_file = set(), [], {}
+ timestamp, eol = None, "\n"
+ out = []
+ for line in lines:
+ body = line.rstrip("\r\n")
+ cols = body.split("\t")
+ if len(cols) == 15 and cols[9] == lang:
+ k = key(cols)
+ seen.add(k)
+ timestamp = timestamp or cols[14]
+ eol = line[len(body):] or eol
+ n, e = new.get(k), en.get(k)
+ if n and k not in skip:
+ updated = list(cols)
+ for i in TEXT_COLUMNS:
+ if n[i] != cols[i] and is_translation(n[i], e[i] if e else
None):
+ updated[i] = n[i]
+ if updated != cols:
+ changed.append(k)
+ line = "\t".join(updated) + line[len(body):]
+ last_of_file[(cols[0], cols[1])] = len(out)
+ out.append(line)
+
+ # New keys go after the last line of the same source file, so the file
stays grouped.
+ added, inserts = [], collections.defaultdict(list)
+ for k, n in new.items():
+ if k in seen or k in skip:
+ continue
+ e = en.get(k)
+ if not any(is_translation(n[i], e[i] if e else None) for i in
TEXT_COLUMNS):
+ continue
+ row = list(n)
+ row[14] = timestamp or row[14]
+ inserts[last_of_file.get((n[0], n[1]), len(out) -
1)].append("\t".join(row) + eol)
+ added.append(k)
+ for pos in sorted(inserts, reverse=True):
+ out[pos + 1:pos + 1] = inserts[pos]
+ return "".join(out), changed, added
+
+
+def run_gsicheck(gsicheck, libdir, path, lang):
+ """Return a Counter of (message, UniqueID) for every error except header
lines."""
+ if not os.access(gsicheck, os.X_OK):
+ sys.exit("gsicheck not found or not executable: %s" % gsicheck)
+ env = dict(os.environ)
+ if libdir:
+ for var in ("DYLD_LIBRARY_PATH", "LD_LIBRARY_PATH"):
+ env[var] = libdir + (os.pathsep + env[var] if env.get(var) else "")
+ with tempfile.TemporaryDirectory() as tmp:
+ shutil.copy(path, os.path.join(tmp, "localize.sdf"))
+ p = subprocess.run([gsicheck, "-c", "-t", "-l", lang, "localize.sdf"],
cwd=tmp, env=env,
+ capture_output=True, text=True, errors="replace")
+ if p.returncode not in (0, 55) and "Error" not in p.stdout + p.stderr:
+ sys.exit("gsicheck failed (%d): %s" % (p.returncode, (p.stdout +
p.stderr).strip()[:500]))
+ errors = collections.Counter()
+ for line in (p.stdout + p.stderr).splitlines():
+ if not line.startswith("Error") or "Format of line is unknown" in line:
+ continue
+ m = re.search(r"UniqueID (.*?): ", line)
+ message = re.sub(r"Line \d+[,:] ?", "", line.split(' "')[0]).strip()
+ errors[(message, m.group(1) if m else None)] += 1
+ return errors
+
+
+def uid_to_key(uid):
+ parts = uid.split("/")
+ gid, lid, helpid, platform = "/".join(parts[3:]).rsplit("/", 3)
+ return (parts[0], parts[1], parts[2], gid, lid, helpid, platform)
+
+
+def write(path, text):
+ with open(path, "w", encoding="utf-8", newline="") as f:
+ f.write(text)
+
+
+def cmd_combine(a):
+ text, stats = combine(a.lang, a.template, a.base, a.newer, a.history)
+ write(a.out, text)
+ print(a.lang, dict(stats))
+
+
+def cmd_merge(a):
+ skip = [tuple(k) for k in json.load(open(a.skip))] if a.skip else []
+ text, changed, added = merge(a.lang, a.tree, a.new, a.template, skip)
+ write(a.out, text)
+ print("%s: %d entries updated, %d keys added" % (a.lang, len(changed),
len(added)))
+
+
+def cmd_gate(a):
+ baseline = run_gsicheck(a.gsicheck, a.libdir, a.tree, a.lang)
+ skip = []
+ for _ in range(a.rounds):
+ text, changed, added = merge(a.lang, a.tree, a.new, a.template, skip)
+ write(a.out, text)
+ new_errors = run_gsicheck(a.gsicheck, a.libdir, a.out, a.lang) -
baseline
+ keyed = {uid_to_key(uid) for _, uid in new_errors if uid} - set(skip)
+ if not keyed:
+ break
+ skip += sorted(keyed)
+ unkeyed = [msg for msg, uid in new_errors.elements() if not uid]
+ if a.skip_out:
+ with open(a.skip_out, "w", encoding="utf-8") as f:
+ json.dump(skip, f, ensure_ascii=False, indent=1)
+ print("%s: %d entries updated, %d keys added, %d left out" % (a.lang,
len(changed), len(added), len(skip)))
+ for k in skip:
+ print(" left out: %s %s %s" % (k[1], k[3], k[4]))
+ if new_errors:
+ for msg, uid in sorted(new_errors):
+ print(" new gsicheck error: %s %s" % (msg, uid or ""))
+ sys.exit(1)
+
+
+def main():
+ p = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
+ sub = p.add_subparsers(dest="command", required=True)
+
+ c = sub.add_parser("combine", help="combine a base and a newer po2oo SDF")
+ c.add_argument("--lang", required=True)
+ c.add_argument("--template", required=True, help="en-US.sdf used for
po2oo")
+ c.add_argument("--base", required=True, help="po2oo output from the
trusted snapshot")
+ c.add_argument("--newer", required=True, help="po2oo output from the newer
snapshot")
+ c.add_argument("--history", nargs="*", default=[], help="po2oo output from
older snapshots")
+ c.add_argument("--out", required=True)
+ c.set_defaults(func=cmd_combine)
+
+ for name, func, helptext in (("merge", cmd_merge, "merge without
checking"),
+ ("gate", cmd_gate, "merge and gate on
gsicheck")):
+ m = sub.add_parser(name, help=helptext)
+ m.add_argument("--lang", required=True)
+ m.add_argument("--tree", required=True,
help="extras/l10n/source/<lang>/localize.sdf")
+ m.add_argument("--new", required=True, help="po2oo (or combine)
output")
+ m.add_argument("--template", required=True, help="en-US.sdf used for
po2oo")
+ m.add_argument("--out", required=True)
+ if name == "merge":
+ m.add_argument("--skip", help="JSON list of keys to leave as they
are")
+ else:
+ m.add_argument("--gsicheck", required=True, help="path to the
gsicheck binary")
+ m.add_argument("--libdir", help="directory with libtl and friends,
e.g. solver/<ver>/<platform>/lib")
+ m.add_argument("--rounds", type=int, default=4)
+ m.add_argument("--skip-out", help="write the left-out keys here as
JSON")
+ m.set_defaults(func=func)
+
+ a = p.parse_args()
+ a.func(a)
+
+
+if __name__ == "__main__":
+ main()
