Santhosh has uploaded a new change for review.
https://gerrit.wikimedia.org/r/170000
Change subject: Revert "Derive MT subsentence map from uppercasing"
......................................................................
Revert "Derive MT subsentence map from uppercasing"
This reverts commit 78fc396b491f16ab566b9d93940b1767be1a9a89.
Commit introduce a regression- Sections with lists are not translated
Change-Id: I6d4b6e99b409ca863b9cd3221bb6118c5752e9ad
---
D bin/apertium-xhtml
M index.js
M lineardoc/LinearDoc.js
M mt/Apertium.js
A tests/apertium/Apertium.test.js
M tests/index.js
M tests/lineardoc/LinearDoc.test.js
M tests/lineardoc/translate.test.json
D tests/mt/Apertium.test.js
9 files changed, 152 insertions(+), 508 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/cxserver
refs/changes/00/170000/1
diff --git a/bin/apertium-xhtml b/bin/apertium-xhtml
deleted file mode 100755
index 08d6865..0000000
--- a/bin/apertium-xhtml
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env node
-var sourceLang, targetLang, sourceHtml, script, args,
- Apertium = require( __dirname + '/../mt/Apertium.js' );
-
-script = process.argv[ 1 ];
-args = process.argv.slice( 2 );
-if ( args.length !== 2 ) {
- process.stderr.write(
- 'Usage: node ' + script + ' <sourceLang> <targetLang> <
xhtmlSource\n\n' +
- 'xhtml must be wrapped in a block element such as <p>...</p> or
<div>..</div>.\n' +
- 'Example:\n\techo "<p>A <b>red</b> box.</p>" | node ' + script
+ ' en es\n\n'
- );
- process.exit( 1 );
-}
-
-sourceLang = args[ 0 ];
-targetLang = args[ 1 ];
-
-sourceHtml = [];
-
-process.stdin.on( 'data', function( data ) {
- sourceHtml.push( data );
-} );
-process.stdin.on( 'end', function() {
- Apertium.translate(
- sourceLang,
- targetLang,
- sourceHtml.join( '' )
- ).then( function( targetHtml ) {
- process.stdout.write( targetHtml + '\n' );
- }, function( error ) {
- console.error( 'error', error );
- process.exit( 2 );
- } );
-} );
diff --git a/index.js b/index.js
index f955bdb..1b309e1 100644
--- a/index.js
+++ b/index.js
@@ -1,6 +1,5 @@
module.exports = {
Segmenter: require( './segmentation/CXSegmenter.js' ).CXSegmenter,
- Apertium: require( './mt/Apertium.js' ),
LinearDoc: require( './lineardoc/LinearDoc.js' ),
Dictionary: require( './dictionary' )
};
diff --git a/lineardoc/LinearDoc.js b/lineardoc/LinearDoc.js
index 3144bad..58a0cdd 100644
--- a/lineardoc/LinearDoc.js
+++ b/lineardoc/LinearDoc.js
@@ -109,7 +109,7 @@
*
* @private
* @param {Object[]} tagArray SAX open tags
- * @return {string[]} Tag names
+ * @returns [string[]] Tag names
*/
function dumpTags( tagArray ) {
var i, len, tag, attr, attrDumps,
@@ -153,7 +153,7 @@
*
* @private
* @param {string} tagName The name of the tag (lowercase)
- * @return {boolean} Whether the tag is an inline empty tag
+ * @returns {boolean} Whether the tag is an inline empty tag
*/
function isInlineEmptyTag( tagName ) {
// link/meta as they're allowed anywhere in HTML5+RDFa, and must be
treated as void
@@ -166,7 +166,7 @@
*
* @private
* @param {string} tagName The name of the tag (lowercase)
- * @return {boolean} Whether the tag is an inline annotation
+ * @returns {boolean} Whether the tag is an inline annotation
*/
isInlineAnnotationTag = ( function ( tagArray ) {
var i, len,
@@ -215,7 +215,7 @@
* @param {number[]} boundaries Boundary offsets
* @param chunks Chunks to which the boundaries apply
* @param {Function} getLength Function returning the length of a chunk
- * @return {Object[]} Array of {chunk: ch, boundaries: [...]}
+ * @returns {Object[]} Array of {chunk: ch, boundaries: [...]}
*/
function getChunkBoundaryGroups( boundaries, chunks, getLength ) {
var i, len, groupBoundaries, chunk, chunkLength, boundary,
@@ -282,28 +282,13 @@
function TextBlock( textChunks ) {
var i, len, cursor;
this.textChunks = textChunks;
- this.offsets = [];
+ this.startOffsets = [];
cursor = 0;
for ( i = 0, len = this.textChunks.length; i < len; i++ ) {
- this.offsets[ i ] = { start: cursor, length: this.textChunks[ i
].text.length };
- cursor += this.offsets[ i ].length;
+ this.startOffsets[i] = cursor;
+ cursor += this.textChunks[i].text.length;
}
}
-
-/**
- * Get the start and length of each non-common annotation
- * @return {Object[]}
- * @return[i].start {number} Position of each text chunk
- * @return[i].length {number} Length of each text chunk
- */
-TextBlock.prototype.getTagOffsets = function () {
- var textBlock = this,
- commonTags = this.getCommonTags();
- return this.offsets.filter( function ( offset, i ) {
- var textChunk = textBlock.textChunks[ i ];
- return textChunk.tags.length > commonTags.length &&
textChunk.text.length > 0;
- } );
-};
/**
* Get the (last) text chunk at a given char offset
@@ -315,37 +300,11 @@
// TODO: bisecting instead of linear search
var i, len;
for ( i = 0, len = this.textChunks.length - 1; i < len; i++ ) {
- if ( this.offsets[ i + 1 ].start > charOffset ) {
+ if ( this.startOffsets[ i + 1 ] > charOffset ) {
break;
}
}
return this.textChunks[ i ];
-};
-
-/**
- * Returns the list of SAX tags that apply to the whole text block
- * @return {Object[]} List of common SAX tags
- */
-TextBlock.prototype.getCommonTags = function () {
- var i, iLen, j, jLen, commonTags, tags;
- if ( this.textChunks.length === 0 ) {
- return [];
- }
- commonTags = this.textChunks[ 0 ].tags.slice();
- for ( i = 0, iLen = this.textChunks.length; i < iLen; i++ ) {
- tags = this.textChunks[ i ].tags;
- if ( tags.length < commonTags.length ) {
- commonTags.splice( tags.length );
- }
- for ( j = 0, jLen = commonTags.length; j < jLen; j++ ) {
- if ( commonTags[ j ] !== tags[ j ] ) {
- // truncate
- commonTags.splice( j );
- break;
- }
- }
- }
- return commonTags;
};
/**
@@ -354,11 +313,11 @@
* @method
* @param {string} targetText Translated plain text
* @param {Object[]} rangeMappings Array of source-target range index mappings
- * @return {TextBlock} Translated textblock with tags applied
+ * @returns {TextBlock} Translated textblock with annotations applied
*/
-TextBlock.prototype.translateTags = function ( targetText, rangeMappings ) {
+TextBlock.prototype.translateAnnotations = function ( targetText,
rangeMappings ) {
var i, iLen, j, rangeMapping, sourceTextChunk, text, pos, textChunk,
offset,
- sourceRangeEnd, targetRangeEnd, tail, tailSpace, commonTags,
+ sourceRangeEnd, targetRangeEnd, tail, tailSpace,
// map of { offset: x, textChunks: [...] }
emptyTextChunks = {},
emptyTextChunkOffsets = [],
@@ -379,7 +338,7 @@
// Create map of empty text chunks, by offset
for ( i = 0, iLen = this.textChunks.length; i < iLen; i++ ) {
textChunk = this.textChunks[ i ];
- offset = this.offsets[ i ].start;
+ offset = this.startOffsets[ i ];
if ( textChunk.text.length > 0 ) {
continue;
}
@@ -394,7 +353,7 @@
emptyTextChunkOffsets.sort( function ( a, b ) { return a - b; } );
for ( i = 0, iLen = rangeMappings.length; i < iLen; i++ ) {
- // Copy tags from source text start offset
+ // Copy annotations from source text start offset
rangeMapping = rangeMappings[ i ];
sourceRangeEnd = rangeMapping.source.start +
rangeMapping.source.length;
targetRangeEnd = rangeMapping.target.start +
rangeMapping.target.length;
@@ -434,9 +393,8 @@
textChunks.sort( function ( textChunk1, textChunk2 ) {
return textChunk1.start - textChunk2.start;
} );
- // Fill in any textChunk gaps using text with commonTags
+ // Fill in any textChunk gaps with unannotated text
pos = 0;
- commonTags = this.getCommonTags();
for ( i = 0, iLen = textChunks.length; i < iLen; i++ ) {
textChunk = textChunks[ i ];
if ( textChunk.start < pos ) {
@@ -448,7 +406,7 @@
length: textChunk.start - pos,
textChunk: new TextChunk(
targetText.substr( pos, textChunk.start
- pos ),
- commonTags
+ []
)
} );
i++;
@@ -465,11 +423,11 @@
}
if ( tail ) {
- // Append tail as text with commonTags
+ // Append tail as unannotated text
textChunks.push( {
start: pos,
length: tail.length,
- textChunk: new TextChunk( tail, commonTags )
+ textChunk: new TextChunk( tail, [] )
} );
pos += tail.length;
}
@@ -480,11 +438,11 @@
pushEmptyTextChunks( pos, emptyTextChunks[ offset ] );
}
if ( tailSpace ) {
- // Append tailSpace as text with commonTags
+ // Append tailSpace as unannotated text
textChunks.push( {
start: pos,
length: tailSpace.length,
- textChunk: new TextChunk( tailSpace, commonTags )
+ textChunk: new TextChunk( tailSpace, [] )
} );
pos += tail.length;
}
diff --git a/mt/Apertium.js b/mt/Apertium.js
index c7f3acf..3d53c92 100644
--- a/mt/Apertium.js
+++ b/mt/Apertium.js
@@ -3,22 +3,16 @@
request = require( 'request' ),
conf = require( __dirname + '/../utils/Conf.js' ),
LinearDoc = require( '../lineardoc/LinearDoc' ),
- //logger = require( '../utils/Logger.js' ),
+ Entities = require( 'html-entities' ).AllHtmlEntities,
+ logger = require( '../utils/Logger.js' ),
+ spawn = require( 'child_process' ).spawn,
// TODO: Tokenize properly. These work for English/Spanish/Catalan
TOKENS =
/[\wáàçéèíïóòúüñÁÀÇÉÈÍÏÓÒÚÜÑ]+(?:[·'][\wáàçéèíïóòúüñÁÀÇÉÈÍÏÓÒÚÜÑ]+)?|[^\wáàçéèíïóòúüñÁÀÇÉÈÍÏÓÒÚÜÑ]+/g,
IS_WORD =
/^[\wáàçéèíïóòúüñÁÀÇÉÈÍÏÓÒÚÜÑ]+(?:[·'][\wáàçéèíïóòúüñÁÀÇÉÈÍÏÓÒÚÜÑ]+)?$/;
apertiumLangMapping = require( __dirname + '/mappings.js' );
-/**
- * Split text into tokens
- * @param {string} lang Language code
- * @param {string} text Text to split
- * @return {Object[]} List of tokens
- * @return[].text Text of the token
- * @return[].isWord Whether the token is a word
- */
-function getTokens( lang, text ) {
+function getTokens( text ) {
// TODO: implement for other languages than English/Spanish/Catalan
return text.match( TOKENS ).map( function ( tokenText ) {
return {
@@ -28,166 +22,114 @@
} );
}
-/**
- * Language-aware uppercasing
- * @param {string} lang Language code
- * @param {string} text Text to uppercase
- * @return {string} Upper-cased text (possibly identical)
- */
-function toUpperCase( lang, text ) {
- // stub: just use the javascript ASCII method for now
- return text.toUpperCase();
+function getRangedText( text ) {
+ var i = 0;
+ return getTokens( text ).map( function ( token ) {
+ return '<span id="mtToken' + ( i++ ) + '">' +
+ LinearDoc.esc( token.text ) +
+ '</span>';
+ } ).join( '' );
}
-/**
- * Create variants of the text, with a different annotation uppercased in each.
- * @param {string} lang Language code
- * @param {string} text Text
- * @param {Object[]} annotationOffsets start and length of each annotation
- * @return {Object[]}
- * @return[].start {number} Start offset of uppercasing
- * @return[].length {number} Length of uppercasing
- * @return[].text {string} Text variant with uppercasing
- */
-function getCaseVariants( lang, sourceText, annotationOffsets ) {
- var i, len, offset, chunk, upperChunk, variantText,
- caseVariants = [];
-
- for ( i = 0, len = annotationOffsets.length; i < len; i++ ) {
- offset = annotationOffsets[ i ];
- chunk = sourceText.slice( offset.start, offset.start +
offset.length );
- upperChunk = toUpperCase( lang, chunk );
- if ( upperChunk === chunk ) {
- // Already uppercased; can't detect change
+function readRangedText( text ) {
+ var i, len, part, match, partText,
+ offset = 0,
+ ranges = {},
+ entities = new Entities(),
+ parts = text.match( /<span id="mtToken\d+">[^<>]+<\/span>/g );
+ for ( i = 0, len = parts.length; i < len; i++ ) {
+ part = parts[ i ];
+ match = part.match( /^<span
id="mtToken(\d+)">([^<>]+)<\/span>$/ );
+ if ( !match ) {
+ console.warn( 'Bad part: "' + part + '"' );
continue;
}
- variantText = [
- sourceText.slice( 0, offset.start ),
- upperChunk,
- sourceText.slice( offset.start + offset.length )
- ].join( '' );
- caseVariants.push( {
- start: offset.start,
- length: offset.length,
- text: variantText
- } );
- }
- return caseVariants;
-}
-
-/**
- * Finds offsets of ranges at which tokens have changed to uppercase
- * @param {string} text Original text
- * @param {string} text Changed text
- * @return {Object[]} start and length for each changed range
- */
-function getChangedCaseRanges( lang, originalText, changedText ) {
- var orig, upper, changed, len, ranges, start, startChar, end, endChar;
- orig = getTokens( lang, originalText );
- upper = getTokens( lang, toUpperCase( lang, originalText ) );
- changed = getTokens( lang, changedText );
-
- len = orig.length;
- if ( len !== upper.length || len !== changed.length ) {
- throw new Error( 'token length mismatch' );
- }
-
- // Find start/end of changed text token ranges. Track char ranges too,
and store these.
- ranges = [];
- // start token
- start = 0;
- // start char
- startChar = 0;
-
- while ( true ) {
- // Skip to first changed word token
- while ( start < len && (
- !( orig[ start ].isWord ) ||
- (
- orig[ start ].text === changed[ start ].text ||
- upper[ start ].text !== changed[ start ].text
- )
- ) ) {
- startChar += orig[ start ].text.length;
- start++;
- }
- if ( start >= len ) {
- break;
- }
- // Find last consecutive changed non-word token
- end = start;
- endChar = startChar + orig[ end ].text.length;
-
- while ( end < len && (
- !( orig[ end ].isWord ) ||
- (
- orig[ end ].text !== upper[ end ].text &&
- upper[ end ].text === changed[ end ].text
- )
- ) ) {
- end++;
- if ( end < len ) {
- endChar += orig[ end ].text.length;
- }
- }
- do {
- if ( end < len ) {
- endChar -= orig[ end ].text.length;
- }
- end--;
- } while ( !( orig[ end ].isWord ) );
- // Store ranges
- ranges.push( {
- start: startChar,
- length: endChar - startChar
- } );
- start = end + 1;
- startChar = endChar;
+ partText = entities.decode( match[ 2 ] );
+ ranges[ match[ 1 ] ] = {
+ text: partText,
+ start: offset,
+ length: partText.length
+ };
+ offset += partText.length;
}
return ranges;
}
-/**
- * Calculate range mappings based on the target text variants
- * @param {string} targetLang The target language
- * @param {Object[]} sourceVariants The start and length of each variation
- * @param {
- * @param {Object} annotationOffsets The start and length of each offset, by
sourceVariantId
- */
-function getRangeMappings( targetLang, sourceVariants, targetText, targetLines
) {
- var i, iLen, j, jLen, changedCaseRanges, sourceRange,
+function getTextAndRangeMappings( rangedSourceText, rangedTargetText ) {
+ var sourceParts, targetParts, ids, i, len, id,
+ targetTextParts = [],
rangeMappings = [];
- if ( sourceVariants.length !== targetLines.length ) {
- throw new Error( 'Translation variants length mismatch' );
- }
- for ( i = 0, iLen = sourceVariants.length; i < iLen; i++ ) {
- sourceRange = {
- start: sourceVariants[ i ].start,
- length: sourceVariants[ i ].length
- };
- changedCaseRanges = getChangedCaseRanges(
- targetLang,
- targetText,
- targetLines[ i ]
- );
- for ( j = 0, jLen = changedCaseRanges.length; j < jLen; j++ ) {
- rangeMappings.push( {
- source: sourceRange,
- target: changedCaseRanges[ j ]
- } );
+ sourceParts = readRangedText( rangedSourceText );
+ targetParts = readRangedText( rangedTargetText );
+ ids = Object.keys( sourceParts );
+ ids.sort( function ( x, y ) {
+ return parseInt( x ) - parseInt( y );
+ } );
+ for ( i = 0, len = ids.length; i < len; i++ ) {
+ id = ids[ i ];
+ if ( targetParts[ id ] === undefined ) {
+ continue;
}
+ rangeMappings.push( {
+ source: sourceParts[ id ],
+ target: targetParts[ id ]
+ } );
+ targetTextParts.push( targetParts[ id ].text );
}
- return rangeMappings;
+ return {
+ text: targetTextParts.join( '' ),
+ rangeMappings: rangeMappings
+ };
}
/**
- * Translate plain text with Apertium API
+ * Translate the text
* @param {string} sourceLang Source language code
* @param {string} targetLang Target language code
- * @param {string} sourceText Source language text
- * @return {Object} Deferred promise: Target language text
+ * @param {string} sourceText Source plain text
+ * @return {Object} Deferred promise: Translated plain text
*/
-function translateTextApertium( sourceLang, targetLang, sourceText ) {
+function translateText( sourceLang, targetLang, sourceText ) {
+ var apertium, rangedSourceText, rangedTargetText,
+ rangedTargetTextData = [],
+ deferred = Q.defer();
+ rangedSourceText = getRangedText( sourceText );
+ apertium = spawn(
+ 'python', [ 'mt/apertium.py', sourceLang + '-' + targetLang,
'-u' ], {
+ stdio: 'pipe',
+ env: {
+ PATH: process.env.PATH,
+ LC_ALL: 'en_US.utf8'
+ }
+ }
+ );
+ apertium.stderr.on( 'data', function ( data ) {
+ logger.error( data );
+ } );
+ apertium.stdout.on( 'data', function ( data ) {
+ rangedTargetTextData.push( '' + data );
+ } );
+ apertium.on( 'close', function ( code ) {
+ if ( code !== 0 ) {
+ deferred.reject( new Error( '' + code ) );
+ return;
+ }
+ rangedTargetText = rangedTargetTextData.join( '' );
+ deferred.resolve( getTextAndRangeMappings( rangedSourceText,
rangedTargetText ) );
+ } );
+ apertium.stdin.write( rangedSourceText );
+ apertium.stdin.end();
+ return deferred.promise;
+}
+
+/**
+ * Translate marked up text relying on the MT engine
+ * @param {string} sourceLang Source language code
+ * @param {string} targetLang Target language code
+ * @param {string} sourceHtml Source rich text
+ * @return {Object} Deferred promise: Translated rich text
+ */
+function translate( sourceLang, targetLang, sourceHtml ) {
var deferred = Q.defer(),
postData;
@@ -196,93 +138,22 @@
form: {
markUnknown: 0,
langpair: apertiumLangMapping[ sourceLang ] + '|' +
apertiumLangMapping[ targetLang ],
- format: 'txt',
- q: sourceText
+ q: sourceHtml
}
};
request.post( postData,
function ( error, response, body ) {
- var message;
if ( error ) {
deferred.reject( new Error( error ) );
return;
}
if ( response.statusCode !== 200 ) {
- message = 'Error ' + response.statusCode;
- message += ' sourceText={' + sourceText + '},
body={' + body + '}';
- deferred.reject( new Error( message ) );
+ deferred.reject( new Error( 'Error while
translating content using apertium' ) );
return;
}
deferred.resolve( JSON.parse( body
).responseData.translatedText );
}
);
- return deferred.promise;
-}
-
-/**
- * Translate multiple lines of plaintext with apertium
- * @param {string} sourceLang Source language code
- * @param {string} targetLang Target language code
- * @param {string[]} sourceLines Source plaintext lines
- * @return {Object} Deferred promise: Translated plaintext lines
- */
-function translateLinesApertium( sourceLang, targetLang, sourceLines ) {
- var deferred = Q.defer();
- // Join lines into single string. Separator must break sentences and
pass through unchanged
- translateTextApertium(
- sourceLang,
- targetLang,
- sourceLines.join( '\n.CxServerApertium.\n' )
- ).then( function ( targetLinesText ) {
- var targetText = targetLinesText
- .replace( /^\s+|\s+$/g, '' )
- .split( /\n\.CxServerApertium\.\n/g );
- deferred.resolve( targetText );
- }, function ( error ) {
- deferred.reject( error );
- } );
- return deferred.promise;
-}
-
-/**
- * Translate text, using case variants to map tag offsets
- * @param {string} sourceLang Source language code
- * @param {string} targetLang Target language code
- * @param {string} sourceText Source plain text
- * @param {Object[]} tagOffsets start and length for each annotation chunk
- * @return {Object} Deferred promise: Translated plain text and range mappings
- */
-function translateTextWithTagOffsets( sourceLang, targetLang, sourceText,
tagOffsets ) {
- var sourceVariants, sourceLines, deferred;
- sourceVariants = getCaseVariants( sourceLang, sourceText, tagOffsets );
- sourceLines = sourceVariants.map( function ( variant ) {
- return variant.text;
- } );
- sourceLines.splice( 0, 0, sourceText );
-
- deferred = Q.defer();
- // Call apertium through module.exports, so tests can override it
- // Join segments with a string that will definitely break sentences and
be preserved
- module.exports.translateLinesApertium(
- sourceLang,
- targetLang,
- sourceLines
- ).then( function ( targetLines ) {
- var targetText, rangeMappings;
- targetText = targetLines.splice( 0, 1 )[ 0 ];
- rangeMappings = getRangeMappings(
- targetLang,
- sourceVariants,
- targetText,
- targetLines
- );
- deferred.resolve( {
- text: targetText,
- rangeMappings: rangeMappings
- } );
- }, function ( error ) {
- deferred.reject( error );
- } );
return deferred.promise;
}
@@ -293,7 +164,7 @@
* @param {string} sourceText Source html
* @return {Object} Deferred promise: Translated html
*/
-function translate( sourceLang, targetLang, sourceHtml ) {
+function translateHtml( sourceLang, targetLang, sourceHtml ) {
var i, len, sourceDoc, targetDoc, itemPromises, deferred,
parser = new LinearDoc.Parser();
parser.init();
@@ -302,50 +173,40 @@
// Clone and adapt sourceDoc
targetDoc = new LinearDoc.Doc( sourceDoc.wrapperTag );
itemPromises = [];
-
- function translateItemDeferred( deferred, item ) {
- itemPromises.push( deferred.promise );
- if ( item.type !== 'textblock' ) {
- deferred.resolve( item );
- return;
- }
- translateTextWithTagOffsets(
- sourceLang,
- targetLang,
- item.item.getPlainText(),
- item.item.getTagOffsets()
- ).then( function ( translated ) {
- var newTextBlock;
- try {
- newTextBlock = item.item.translateTags(
- translated.text,
translated.rangeMappings
- );
- deferred.resolve( {
- type: 'textblock',
- item: newTextBlock
- } );
- } catch ( ex ) {
- deferred.reject( ex );
- }
- }, function ( error ) {
- deferred.reject( error );
- } );
- }
for ( i = 0, len = sourceDoc.items.length; i < len; i++ ) {
- translateItemDeferred( Q.defer(), sourceDoc.items[ i ] );
+ /*jshint loopfunc:true */
+ ( function ( deferred, item ) {
+ itemPromises.push( deferred.promise );
+ if ( item.type !== 'textblock' ) {
+ deferred.resolve( item );
+ return;
+ }
+ translateText( sourceLang, targetLang,
item.item.getPlainText() ).then( function ( translated ) {
+ var newTextBlock;
+ try {
+ newTextBlock =
item.item.translateAnnotations(
+ translated.text,
translated.rangeMappings
+ );
+ deferred.resolve( {
+ type: 'textblock',
+ item: newTextBlock
+ } );
+ } catch ( ex ) {
+ deferred.reject( ex );
+ }
+ } );
+ }( Q.defer(), sourceDoc.items[ i ] ) );
}
deferred = Q.defer();
Q.all( itemPromises ).spread( function () {
targetDoc.items = Array.prototype.slice.call( arguments, 0 );
- deferred.resolve( targetDoc.getHtml() );
- }, function ( error ) {
- deferred.reject( error );
+ return deferred.resolve( targetDoc.getHtml() );
} );
return deferred.promise;
}
module.exports = {
translate: translate,
- translateLinesApertium: translateLinesApertium,
- getTokens: getTokens
+ translateHtml: translateHtml,
+ translateText: translateText
};
diff --git a/tests/apertium/Apertium.test.js b/tests/apertium/Apertium.test.js
new file mode 100644
index 0000000..ff9ae93
--- /dev/null
+++ b/tests/apertium/Apertium.test.js
@@ -0,0 +1,8 @@
+var caHtml, Apertium = require( '../../mt/Apertium' );
+
+caHtml = '<p>un dos <b><a href="3">tres</a> quatre <i><a href="5">cinc</a> sis
set</i> vuit nou</b> deu</p>.';
+
+Apertium.translate( 'ca', 'es', caHtml ).then( function ( esHtml ) {
+ console.log( 'caHtml:', caHtml );
+ console.log( 'esHtml:', esHtml );
+} );
diff --git a/tests/index.js b/tests/index.js
index ddf10d2..17ee135 100644
--- a/tests/index.js
+++ b/tests/index.js
@@ -3,7 +3,6 @@
tests = [
'./tests/segmentation/CXSegmenter.test.js',
'./tests/lineardoc/LinearDoc.test.js',
- './tests/mt/Apertium.test.js',
'./tests/dictionary/Dictionary.test.js'
];
diff --git a/tests/lineardoc/LinearDoc.test.js
b/tests/lineardoc/LinearDoc.test.js
index 745fb36..7b09d22 100644
--- a/tests/lineardoc/LinearDoc.test.js
+++ b/tests/lineardoc/LinearDoc.test.js
@@ -32,7 +32,7 @@
}
} );
-QUnit.test( 'LinearDoc translatetags', function ( assert ) {
+QUnit.test( 'LinearDoc translateAnnotations', function ( assert ) {
var parser, textBlock1, textBlock2, i, len, test, doc;
QUnit.expect( 2 * transTests.length );
@@ -49,7 +49,7 @@
test.source,
'Reconstructed source HTML'
);
- textBlock2 = textBlock1.translateTags(
+ textBlock2 = textBlock1.translateAnnotations(
test.targetText,
test.rangeMappings
);
diff --git a/tests/lineardoc/translate.test.json
b/tests/lineardoc/translate.test.json
index 0508548..da48d1c 100644
--- a/tests/lineardoc/translate.test.json
+++ b/tests/lineardoc/translate.test.json
@@ -46,23 +46,6 @@
"expect": "S \n"
},
{
- "source": "Un cotxe blau.",
- "targetText": "A blue car.",
- "rangeMappings": [],
- "expect": "A blue car."
-},
-{
- "source": "Un <a href=\"x\">cotxe</a> blau.",
- "targetText": "A blue car.",
- "rangeMappings": [
- {
- "source": { "start": 3, "length": 5 },
- "target": { "start": 7, "length": 3 }
- }
- ],
- "expect": "A blue <a href=\"x\">car</a>."
-},
-{
"source": "<b>a1 a2</b> <u>b1 b2</u> <i>c1 c2</i>",
"targetText": "C1 A1 B1 B2 A2 C2",
"rangeMappings": [
diff --git a/tests/mt/Apertium.test.js b/tests/mt/Apertium.test.js
deleted file mode 100644
index 7d2c5a9..0000000
--- a/tests/mt/Apertium.test.js
+++ /dev/null
@@ -1,129 +0,0 @@
-QUnit.module( 'Apertium' );
-
-var tests,
- Q = require( 'q' );
-
-// In each case, below, just "source" and "textTranslations" should be
sufficient for linearDoc
-// to derive "target". // The plaintext strings in "textTranslations" are
real Apertium output.
-// They are pre-cached (Apertium is not actually called during the test), so
that:
-// 1. Unit testing does not depend on Apertium
-// 2. Tests will not break if an Apertium upgrade changes some translations
-tests = [
- {
- title: 'All caps words',
- source: '<p>A <b>Japanese</b> <i>BBC</i> article</p>',
- target: '<p>Un artículo de BBC <b>japonés</b></p>',
- textTranslations: {
- 'A Japanese BBC article': 'Un artículo de BBC japonés',
- 'A JAPANESE BBC article': 'Un artículo de BBC JAPONÉS'
- }
- },
- {
- title: 'Title caps one-to-many',
- source: '<div>A <b>modern</b> Britain.</div>',
- target: '<div>Una Gran Bretaña <b>moderna</b>.</div>',
- textTranslations: {
- 'A modern Britain.': 'Una Gran Bretaña moderna.',
- 'A MODERN Britain.': 'Una Gran Bretaña MODERNA.',
- 'A modern BRITAIN.': 'Una GRAN BRETAÑA moderna.'
- }
- },
- {
- title: 'Reordering with nested tags',
- source: '<p>The <b>big <i>red</i></b> dog</p>',
- target: '<p>El perro <b><i>rojo</i></b> <b>grande</b></p>',
- textTranslations: {
- 'The big red dog': 'El perro rojo grande',
- 'The BIG red dog': 'El perro rojo GRANDE',
- 'The big RED dog': 'El perro ROJO grande'
- }
- },
- {
- title: 'Many-to-one with nested tags',
- source: '<p>He said "<i>I tile <a
href="x">bathrooms</a>.</i>"</p>',
- target: '<p>Diga que "<i>enladrillo</i> <i><a
href="x">baños</a></i>."</p>',
- textTranslations: {
- 'He said "I tile bathrooms."': 'Diga que "enladrillo
baños."',
- 'He said "I TILE bathrooms."': 'Diga que "ENLADRILLO
baños."',
- 'He said "I tile BATHROOMS."': 'Diga que "enladrillo
BAÑOS."'
- }
- },
- {
- title: 'Reordering at either ends of a tag',
- source: '<p>The <b>big red</b> dog</p>',
- target: '<p>El perro <b>rojo grande</b></p>',
- textTranslations: {
- 'The big red dog': 'El perro rojo grande',
- 'The BIG RED dog': 'El perro ROJO GRANDE'
- }
- },
- {
- title: 'Identical tags separated by whitespace',
- source: '<p>The <b>big</b> <b>red</b> dog</p>',
- target: '<p>El perro <b>rojo</b> <b>grande</b></p>',
- textTranslations: {
- 'The big red dog': 'El perro rojo grande',
- 'The BIG red dog': 'El perro rojo GRANDE',
- 'The big RED dog': 'El perro ROJO grande'
- }
- },
- {
- title: 'Non-identical links separated by whitespace',
- source: '<p>The <a href="1">big</a> <a href="2">red</a>
dog</p>',
- target: '<p>El perro <a href="2">rojo</a> <a
href="1">grande</a></p>',
- textTranslations: {
- 'The big red dog': 'El perro rojo grande',
- 'The BIG red dog': 'El perro rojo GRANDE',
- 'The big RED dog': 'El perro ROJO grande'
- }
- }
-];
-
-QUnit.test( 'Apertium wrapper tests', function ( assert ) {
- var textTranslations;
-
- // Fake the actual Apertium call
- CX.Apertium.translateLinesApertium = function ( sourceLang, targetLang,
sourceLines ) {
- var deferred = Q.defer();
- setTimeout( function () {
- var targetLines;
- try {
- sourceLines.map( function ( line ) {
- return textTranslations[ line ] || 'X'
+ line + 'X';
- } );
- deferred.resolve( targetLines );
- } catch ( error ) {
- deferred.reject( error );
- }
- } );
- return deferred.promise;
- };
-
- QUnit.expect( tests.length );
-
- function resumeTests( i ) {
- var test;
- if ( i >= tests.length ) {
- return;
- }
- test = tests[ i ];
- textTranslations = test.textTranslations;
-
- QUnit.stop();
-
- CX.Apertium.translate( 'xx', 'yy', test.source ).then( function
( target ) {
- assert.strictEqual(
- target,
- test.target,
- test.title
- );
- QUnit.start();
- resumeTests( i + 1 );
- }, function ( error ) {
- assert.ok( false, test.title + ': ' + error );
- QUnit.start();
- resumeTests( i + 1 );
- } );
- }
- resumeTests( 0 );
-} );
--
To view, visit https://gerrit.wikimedia.org/r/170000
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I6d4b6e99b409ca863b9cd3221bb6118c5752e9ad
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/cxserver
Gerrit-Branch: master
Gerrit-Owner: Santhosh <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
