Divec has uploaded a new change for review.
https://gerrit.wikimedia.org/r/176507
Change subject: Normalize whitespace of each Apertium output line
......................................................................
Normalize whitespace of each Apertium output line
When translateLines pushes multiple strings through Apertium in a single API
call, strip leading/trailing whitespace from each of the returned strings.
Apertium.js already ensured that no leading/trailing whitespace is passed to
Apertium. However, in some cases Apertium can add extraneous leading/trailing
whitespace where none existed before (e.g. translating 'It is' from en to es).
This was causing differently cased versions of the same text to have different
numbers of tokens, which was confusing the case range detection code.
Bug: T76124
Change-Id: I676ad2d80bc6eb99757c984a31ba1f17b2225575
---
M mt/Apertium.js
1 file changed, 9 insertions(+), 5 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/cxserver
refs/changes/07/176507/1
diff --git a/mt/Apertium.js b/mt/Apertium.js
index 1ca46b2..aaf9b56 100644
--- a/mt/Apertium.js
+++ b/mt/Apertium.js
@@ -103,8 +103,12 @@
sourceLang,
targetLang,
trimmedSourceLines
- ).then( function ( trimmedTargetLines ) {
- var targetLines, targetText, rangeMappings;
+ ).then( function ( unnormalizedTargetLines ) {
+ var trimmedTargetLines, targetLines, targetText, rangeMappings;
+ trimmedTargetLines = unnormalizedTargetLines.map( function (
line ) {
+ return line.replace( /^\s+|\s+$/g, '' );
+ } );
+ console.log( trimmedSourceLines, trimmedTargetLines );
targetLines = trimmedTargetLines.map( function (
trimmedTargetLine ) {
return preSpace + trimmedTargetLine + postSpace;
@@ -135,6 +139,8 @@
/**
* Translate multiple lines of plaintext with apertium
+ *
+ * The output may need normalizing for leading/trailing whitespace etc.
* @param {string} sourceLang Source language code
* @param {string} targetLang Target language code
* @param {string[]} sourceLines Source plaintext lines
@@ -152,9 +158,7 @@
targetLang,
sourceLinesText
).then( function ( targetLinesText ) {
- var targetText = targetLinesText
- .replace( /^\s+|\s+$/g, '' )
- .split( /\n\.CxServerApertium\.\n/g );
+ var targetText = targetLinesText.split(
/\n\.CxServerApertium\.\n/g );
deferred.resolve( targetText );
}, function ( error ) {
deferred.reject( error );
--
To view, visit https://gerrit.wikimedia.org/r/176507
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I676ad2d80bc6eb99757c984a31ba1f17b2225575
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/cxserver
Gerrit-Branch: master
Gerrit-Owner: Divec <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
