jenkins-bot has submitted this change and it was merged. Change subject: Derive MT subsentence map from uppercasing ......................................................................
Derive MT subsentence map from uppercasing See https://www.mediawiki.org/wiki/Content_translation/Markup bin/apertium-xhtml * Executable script for testing mt/Apertium.js * Subsentence map detection logic lineardoc/LinearDoc.js * Tag range info functionality tests/* * Unit tests for subsentence mapping Change-Id: I6d1483e69aae979a684728f7724f6874194eee11 --- A bin/apertium-xhtml M index.js M lineardoc/LinearDoc.js M mt/Apertium.js D tests/apertium/Apertium.test.js M tests/index.js M tests/lineardoc/LinearDoc.test.js M tests/lineardoc/translate.test.json A tests/mt/Apertium.test.js 9 files changed, 509 insertions(+), 153 deletions(-) Approvals: Santhosh: Looks good to me, approved jenkins-bot: Verified diff --git a/bin/apertium-xhtml b/bin/apertium-xhtml new file mode 100755 index 0000000..08d6865 --- /dev/null +++ b/bin/apertium-xhtml @@ -0,0 +1,35 @@ +#!/usr/bin/env node +var sourceLang, targetLang, sourceHtml, script, args, + Apertium = require( __dirname + '/../mt/Apertium.js' ); + +script = process.argv[ 1 ]; +args = process.argv.slice( 2 ); +if ( args.length !== 2 ) { + process.stderr.write( + 'Usage: node ' + script + ' <sourceLang> <targetLang> < xhtmlSource\n\n' + + 'xhtml must be wrapped in a block element such as <p>...</p> or <div>..</div>.\n' + + 'Example:\n\techo "<p>A <b>red</b> box.</p>" | node ' + script + ' en es\n\n' + ); + process.exit( 1 ); +} + +sourceLang = args[ 0 ]; +targetLang = args[ 1 ]; + +sourceHtml = []; + +process.stdin.on( 'data', function( data ) { + sourceHtml.push( data ); +} ); +process.stdin.on( 'end', function() { + Apertium.translate( + sourceLang, + targetLang, + sourceHtml.join( '' ) + ).then( function( targetHtml ) { + process.stdout.write( targetHtml + '\n' ); + }, function( error ) { + console.error( 'error', error ); + process.exit( 2 ); + } ); +} ); diff --git a/index.js b/index.js index 1b309e1..f955bdb 100644 --- a/index.js +++ b/index.js @@ -1,5 +1,6 @@ module.exports = { Segmenter: require( './segmentation/CXSegmenter.js' ).CXSegmenter, + Apertium: require( './mt/Apertium.js' ), LinearDoc: require( './lineardoc/LinearDoc.js' ), Dictionary: require( './dictionary' ) }; diff --git a/lineardoc/LinearDoc.js b/lineardoc/LinearDoc.js index 58a0cdd..3144bad 100644 --- a/lineardoc/LinearDoc.js +++ b/lineardoc/LinearDoc.js @@ -109,7 +109,7 @@ * * @private * @param {Object[]} tagArray SAX open tags - * @returns [string[]] Tag names + * @return {string[]} Tag names */ function dumpTags( tagArray ) { var i, len, tag, attr, attrDumps, @@ -153,7 +153,7 @@ * * @private * @param {string} tagName The name of the tag (lowercase) - * @returns {boolean} Whether the tag is an inline empty tag + * @return {boolean} Whether the tag is an inline empty tag */ function isInlineEmptyTag( tagName ) { // link/meta as they're allowed anywhere in HTML5+RDFa, and must be treated as void @@ -166,7 +166,7 @@ * * @private * @param {string} tagName The name of the tag (lowercase) - * @returns {boolean} Whether the tag is an inline annotation + * @return {boolean} Whether the tag is an inline annotation */ isInlineAnnotationTag = ( function ( tagArray ) { var i, len, @@ -215,7 +215,7 @@ * @param {number[]} boundaries Boundary offsets * @param chunks Chunks to which the boundaries apply * @param {Function} getLength Function returning the length of a chunk - * @returns {Object[]} Array of {chunk: ch, boundaries: [...]} + * @return {Object[]} Array of {chunk: ch, boundaries: [...]} */ function getChunkBoundaryGroups( boundaries, chunks, getLength ) { var i, len, groupBoundaries, chunk, chunkLength, boundary, @@ -282,13 +282,28 @@ function TextBlock( textChunks ) { var i, len, cursor; this.textChunks = textChunks; - this.startOffsets = []; + this.offsets = []; cursor = 0; for ( i = 0, len = this.textChunks.length; i < len; i++ ) { - this.startOffsets[i] = cursor; - cursor += this.textChunks[i].text.length; + this.offsets[ i ] = { start: cursor, length: this.textChunks[ i ].text.length }; + cursor += this.offsets[ i ].length; } } + +/** + * Get the start and length of each non-common annotation + * @return {Object[]} + * @return[i].start {number} Position of each text chunk + * @return[i].length {number} Length of each text chunk + */ +TextBlock.prototype.getTagOffsets = function () { + var textBlock = this, + commonTags = this.getCommonTags(); + return this.offsets.filter( function ( offset, i ) { + var textChunk = textBlock.textChunks[ i ]; + return textChunk.tags.length > commonTags.length && textChunk.text.length > 0; + } ); +}; /** * Get the (last) text chunk at a given char offset @@ -300,11 +315,37 @@ // TODO: bisecting instead of linear search var i, len; for ( i = 0, len = this.textChunks.length - 1; i < len; i++ ) { - if ( this.startOffsets[ i + 1 ] > charOffset ) { + if ( this.offsets[ i + 1 ].start > charOffset ) { break; } } return this.textChunks[ i ]; +}; + +/** + * Returns the list of SAX tags that apply to the whole text block + * @return {Object[]} List of common SAX tags + */ +TextBlock.prototype.getCommonTags = function () { + var i, iLen, j, jLen, commonTags, tags; + if ( this.textChunks.length === 0 ) { + return []; + } + commonTags = this.textChunks[ 0 ].tags.slice(); + for ( i = 0, iLen = this.textChunks.length; i < iLen; i++ ) { + tags = this.textChunks[ i ].tags; + if ( tags.length < commonTags.length ) { + commonTags.splice( tags.length ); + } + for ( j = 0, jLen = commonTags.length; j < jLen; j++ ) { + if ( commonTags[ j ] !== tags[ j ] ) { + // truncate + commonTags.splice( j ); + break; + } + } + } + return commonTags; }; /** @@ -313,11 +354,11 @@ * @method * @param {string} targetText Translated plain text * @param {Object[]} rangeMappings Array of source-target range index mappings - * @returns {TextBlock} Translated textblock with annotations applied + * @return {TextBlock} Translated textblock with tags applied */ -TextBlock.prototype.translateAnnotations = function ( targetText, rangeMappings ) { +TextBlock.prototype.translateTags = function ( targetText, rangeMappings ) { var i, iLen, j, rangeMapping, sourceTextChunk, text, pos, textChunk, offset, - sourceRangeEnd, targetRangeEnd, tail, tailSpace, + sourceRangeEnd, targetRangeEnd, tail, tailSpace, commonTags, // map of { offset: x, textChunks: [...] } emptyTextChunks = {}, emptyTextChunkOffsets = [], @@ -338,7 +379,7 @@ // Create map of empty text chunks, by offset for ( i = 0, iLen = this.textChunks.length; i < iLen; i++ ) { textChunk = this.textChunks[ i ]; - offset = this.startOffsets[ i ]; + offset = this.offsets[ i ].start; if ( textChunk.text.length > 0 ) { continue; } @@ -353,7 +394,7 @@ emptyTextChunkOffsets.sort( function ( a, b ) { return a - b; } ); for ( i = 0, iLen = rangeMappings.length; i < iLen; i++ ) { - // Copy annotations from source text start offset + // Copy tags from source text start offset rangeMapping = rangeMappings[ i ]; sourceRangeEnd = rangeMapping.source.start + rangeMapping.source.length; targetRangeEnd = rangeMapping.target.start + rangeMapping.target.length; @@ -393,8 +434,9 @@ textChunks.sort( function ( textChunk1, textChunk2 ) { return textChunk1.start - textChunk2.start; } ); - // Fill in any textChunk gaps with unannotated text + // Fill in any textChunk gaps using text with commonTags pos = 0; + commonTags = this.getCommonTags(); for ( i = 0, iLen = textChunks.length; i < iLen; i++ ) { textChunk = textChunks[ i ]; if ( textChunk.start < pos ) { @@ -406,7 +448,7 @@ length: textChunk.start - pos, textChunk: new TextChunk( targetText.substr( pos, textChunk.start - pos ), - [] + commonTags ) } ); i++; @@ -423,11 +465,11 @@ } if ( tail ) { - // Append tail as unannotated text + // Append tail as text with commonTags textChunks.push( { start: pos, length: tail.length, - textChunk: new TextChunk( tail, [] ) + textChunk: new TextChunk( tail, commonTags ) } ); pos += tail.length; } @@ -438,11 +480,11 @@ pushEmptyTextChunks( pos, emptyTextChunks[ offset ] ); } if ( tailSpace ) { - // Append tailSpace as unannotated text + // Append tailSpace as text with commonTags textChunks.push( { start: pos, length: tailSpace.length, - textChunk: new TextChunk( tailSpace, [] ) + textChunk: new TextChunk( tailSpace, commonTags ) } ); pos += tail.length; } diff --git a/mt/Apertium.js b/mt/Apertium.js index 3d53c92..c7f3acf 100644 --- a/mt/Apertium.js +++ b/mt/Apertium.js @@ -3,16 +3,22 @@ request = require( 'request' ), conf = require( __dirname + '/../utils/Conf.js' ), LinearDoc = require( '../lineardoc/LinearDoc' ), - Entities = require( 'html-entities' ).AllHtmlEntities, - logger = require( '../utils/Logger.js' ), - spawn = require( 'child_process' ).spawn, + //logger = require( '../utils/Logger.js' ), // TODO: Tokenize properly. These work for English/Spanish/Catalan TOKENS = /[\wáàçéèíïóòúüñÁÀÇÉÈÍÏÓÒÚÜÑ]+(?:[·'][\wáàçéèíïóòúüñÁÀÇÉÈÍÏÓÒÚÜÑ]+)?|[^\wáàçéèíïóòúüñÁÀÇÉÈÍÏÓÒÚÜÑ]+/g, IS_WORD = /^[\wáàçéèíïóòúüñÁÀÇÉÈÍÏÓÒÚÜÑ]+(?:[·'][\wáàçéèíïóòúüñÁÀÇÉÈÍÏÓÒÚÜÑ]+)?$/; apertiumLangMapping = require( __dirname + '/mappings.js' ); -function getTokens( text ) { +/** + * Split text into tokens + * @param {string} lang Language code + * @param {string} text Text to split + * @return {Object[]} List of tokens + * @return[].text Text of the token + * @return[].isWord Whether the token is a word + */ +function getTokens( lang, text ) { // TODO: implement for other languages than English/Spanish/Catalan return text.match( TOKENS ).map( function ( tokenText ) { return { @@ -22,114 +28,166 @@ } ); } -function getRangedText( text ) { - var i = 0; - return getTokens( text ).map( function ( token ) { - return '<span id="mtToken' + ( i++ ) + '">' + - LinearDoc.esc( token.text ) + - '</span>'; - } ).join( '' ); +/** + * Language-aware uppercasing + * @param {string} lang Language code + * @param {string} text Text to uppercase + * @return {string} Upper-cased text (possibly identical) + */ +function toUpperCase( lang, text ) { + // stub: just use the javascript ASCII method for now + return text.toUpperCase(); } -function readRangedText( text ) { - var i, len, part, match, partText, - offset = 0, - ranges = {}, - entities = new Entities(), - parts = text.match( /<span id="mtToken\d+">[^<>]+<\/span>/g ); - for ( i = 0, len = parts.length; i < len; i++ ) { - part = parts[ i ]; - match = part.match( /^<span id="mtToken(\d+)">([^<>]+)<\/span>$/ ); - if ( !match ) { - console.warn( 'Bad part: "' + part + '"' ); +/** + * Create variants of the text, with a different annotation uppercased in each. + * @param {string} lang Language code + * @param {string} text Text + * @param {Object[]} annotationOffsets start and length of each annotation + * @return {Object[]} + * @return[].start {number} Start offset of uppercasing + * @return[].length {number} Length of uppercasing + * @return[].text {string} Text variant with uppercasing + */ +function getCaseVariants( lang, sourceText, annotationOffsets ) { + var i, len, offset, chunk, upperChunk, variantText, + caseVariants = []; + + for ( i = 0, len = annotationOffsets.length; i < len; i++ ) { + offset = annotationOffsets[ i ]; + chunk = sourceText.slice( offset.start, offset.start + offset.length ); + upperChunk = toUpperCase( lang, chunk ); + if ( upperChunk === chunk ) { + // Already uppercased; can't detect change continue; } - partText = entities.decode( match[ 2 ] ); - ranges[ match[ 1 ] ] = { - text: partText, - start: offset, - length: partText.length - }; - offset += partText.length; + variantText = [ + sourceText.slice( 0, offset.start ), + upperChunk, + sourceText.slice( offset.start + offset.length ) + ].join( '' ); + caseVariants.push( { + start: offset.start, + length: offset.length, + text: variantText + } ); + } + return caseVariants; +} + +/** + * Finds offsets of ranges at which tokens have changed to uppercase + * @param {string} text Original text + * @param {string} text Changed text + * @return {Object[]} start and length for each changed range + */ +function getChangedCaseRanges( lang, originalText, changedText ) { + var orig, upper, changed, len, ranges, start, startChar, end, endChar; + orig = getTokens( lang, originalText ); + upper = getTokens( lang, toUpperCase( lang, originalText ) ); + changed = getTokens( lang, changedText ); + + len = orig.length; + if ( len !== upper.length || len !== changed.length ) { + throw new Error( 'token length mismatch' ); + } + + // Find start/end of changed text token ranges. Track char ranges too, and store these. + ranges = []; + // start token + start = 0; + // start char + startChar = 0; + + while ( true ) { + // Skip to first changed word token + while ( start < len && ( + !( orig[ start ].isWord ) || + ( + orig[ start ].text === changed[ start ].text || + upper[ start ].text !== changed[ start ].text + ) + ) ) { + startChar += orig[ start ].text.length; + start++; + } + if ( start >= len ) { + break; + } + // Find last consecutive changed non-word token + end = start; + endChar = startChar + orig[ end ].text.length; + + while ( end < len && ( + !( orig[ end ].isWord ) || + ( + orig[ end ].text !== upper[ end ].text && + upper[ end ].text === changed[ end ].text + ) + ) ) { + end++; + if ( end < len ) { + endChar += orig[ end ].text.length; + } + } + do { + if ( end < len ) { + endChar -= orig[ end ].text.length; + } + end--; + } while ( !( orig[ end ].isWord ) ); + // Store ranges + ranges.push( { + start: startChar, + length: endChar - startChar + } ); + start = end + 1; + startChar = endChar; } return ranges; } -function getTextAndRangeMappings( rangedSourceText, rangedTargetText ) { - var sourceParts, targetParts, ids, i, len, id, - targetTextParts = [], +/** + * Calculate range mappings based on the target text variants + * @param {string} targetLang The target language + * @param {Object[]} sourceVariants The start and length of each variation + * @param { + * @param {Object} annotationOffsets The start and length of each offset, by sourceVariantId + */ +function getRangeMappings( targetLang, sourceVariants, targetText, targetLines ) { + var i, iLen, j, jLen, changedCaseRanges, sourceRange, rangeMappings = []; - sourceParts = readRangedText( rangedSourceText ); - targetParts = readRangedText( rangedTargetText ); - ids = Object.keys( sourceParts ); - ids.sort( function ( x, y ) { - return parseInt( x ) - parseInt( y ); - } ); - for ( i = 0, len = ids.length; i < len; i++ ) { - id = ids[ i ]; - if ( targetParts[ id ] === undefined ) { - continue; - } - rangeMappings.push( { - source: sourceParts[ id ], - target: targetParts[ id ] - } ); - targetTextParts.push( targetParts[ id ].text ); + if ( sourceVariants.length !== targetLines.length ) { + throw new Error( 'Translation variants length mismatch' ); } - return { - text: targetTextParts.join( '' ), - rangeMappings: rangeMappings - }; + for ( i = 0, iLen = sourceVariants.length; i < iLen; i++ ) { + sourceRange = { + start: sourceVariants[ i ].start, + length: sourceVariants[ i ].length + }; + changedCaseRanges = getChangedCaseRanges( + targetLang, + targetText, + targetLines[ i ] + ); + for ( j = 0, jLen = changedCaseRanges.length; j < jLen; j++ ) { + rangeMappings.push( { + source: sourceRange, + target: changedCaseRanges[ j ] + } ); + } + } + return rangeMappings; } /** - * Translate the text + * Translate plain text with Apertium API * @param {string} sourceLang Source language code * @param {string} targetLang Target language code - * @param {string} sourceText Source plain text - * @return {Object} Deferred promise: Translated plain text + * @param {string} sourceText Source language text + * @return {Object} Deferred promise: Target language text */ -function translateText( sourceLang, targetLang, sourceText ) { - var apertium, rangedSourceText, rangedTargetText, - rangedTargetTextData = [], - deferred = Q.defer(); - rangedSourceText = getRangedText( sourceText ); - apertium = spawn( - 'python', [ 'mt/apertium.py', sourceLang + '-' + targetLang, '-u' ], { - stdio: 'pipe', - env: { - PATH: process.env.PATH, - LC_ALL: 'en_US.utf8' - } - } - ); - apertium.stderr.on( 'data', function ( data ) { - logger.error( data ); - } ); - apertium.stdout.on( 'data', function ( data ) { - rangedTargetTextData.push( '' + data ); - } ); - apertium.on( 'close', function ( code ) { - if ( code !== 0 ) { - deferred.reject( new Error( '' + code ) ); - return; - } - rangedTargetText = rangedTargetTextData.join( '' ); - deferred.resolve( getTextAndRangeMappings( rangedSourceText, rangedTargetText ) ); - } ); - apertium.stdin.write( rangedSourceText ); - apertium.stdin.end(); - return deferred.promise; -} - -/** - * Translate marked up text relying on the MT engine - * @param {string} sourceLang Source language code - * @param {string} targetLang Target language code - * @param {string} sourceHtml Source rich text - * @return {Object} Deferred promise: Translated rich text - */ -function translate( sourceLang, targetLang, sourceHtml ) { +function translateTextApertium( sourceLang, targetLang, sourceText ) { var deferred = Q.defer(), postData; @@ -138,22 +196,93 @@ form: { markUnknown: 0, langpair: apertiumLangMapping[ sourceLang ] + '|' + apertiumLangMapping[ targetLang ], - q: sourceHtml + format: 'txt', + q: sourceText } }; request.post( postData, function ( error, response, body ) { + var message; if ( error ) { deferred.reject( new Error( error ) ); return; } if ( response.statusCode !== 200 ) { - deferred.reject( new Error( 'Error while translating content using apertium' ) ); + message = 'Error ' + response.statusCode; + message += ' sourceText={' + sourceText + '}, body={' + body + '}'; + deferred.reject( new Error( message ) ); return; } deferred.resolve( JSON.parse( body ).responseData.translatedText ); } ); + return deferred.promise; +} + +/** + * Translate multiple lines of plaintext with apertium + * @param {string} sourceLang Source language code + * @param {string} targetLang Target language code + * @param {string[]} sourceLines Source plaintext lines + * @return {Object} Deferred promise: Translated plaintext lines + */ +function translateLinesApertium( sourceLang, targetLang, sourceLines ) { + var deferred = Q.defer(); + // Join lines into single string. Separator must break sentences and pass through unchanged + translateTextApertium( + sourceLang, + targetLang, + sourceLines.join( '\n.CxServerApertium.\n' ) + ).then( function ( targetLinesText ) { + var targetText = targetLinesText + .replace( /^\s+|\s+$/g, '' ) + .split( /\n\.CxServerApertium\.\n/g ); + deferred.resolve( targetText ); + }, function ( error ) { + deferred.reject( error ); + } ); + return deferred.promise; +} + +/** + * Translate text, using case variants to map tag offsets + * @param {string} sourceLang Source language code + * @param {string} targetLang Target language code + * @param {string} sourceText Source plain text + * @param {Object[]} tagOffsets start and length for each annotation chunk + * @return {Object} Deferred promise: Translated plain text and range mappings + */ +function translateTextWithTagOffsets( sourceLang, targetLang, sourceText, tagOffsets ) { + var sourceVariants, sourceLines, deferred; + sourceVariants = getCaseVariants( sourceLang, sourceText, tagOffsets ); + sourceLines = sourceVariants.map( function ( variant ) { + return variant.text; + } ); + sourceLines.splice( 0, 0, sourceText ); + + deferred = Q.defer(); + // Call apertium through module.exports, so tests can override it + // Join segments with a string that will definitely break sentences and be preserved + module.exports.translateLinesApertium( + sourceLang, + targetLang, + sourceLines + ).then( function ( targetLines ) { + var targetText, rangeMappings; + targetText = targetLines.splice( 0, 1 )[ 0 ]; + rangeMappings = getRangeMappings( + targetLang, + sourceVariants, + targetText, + targetLines + ); + deferred.resolve( { + text: targetText, + rangeMappings: rangeMappings + } ); + }, function ( error ) { + deferred.reject( error ); + } ); return deferred.promise; } @@ -164,7 +293,7 @@ * @param {string} sourceText Source html * @return {Object} Deferred promise: Translated html */ -function translateHtml( sourceLang, targetLang, sourceHtml ) { +function translate( sourceLang, targetLang, sourceHtml ) { var i, len, sourceDoc, targetDoc, itemPromises, deferred, parser = new LinearDoc.Parser(); parser.init(); @@ -173,40 +302,50 @@ // Clone and adapt sourceDoc targetDoc = new LinearDoc.Doc( sourceDoc.wrapperTag ); itemPromises = []; - for ( i = 0, len = sourceDoc.items.length; i < len; i++ ) { - /*jshint loopfunc:true */ - ( function ( deferred, item ) { - itemPromises.push( deferred.promise ); - if ( item.type !== 'textblock' ) { - deferred.resolve( item ); - return; + + function translateItemDeferred( deferred, item ) { + itemPromises.push( deferred.promise ); + if ( item.type !== 'textblock' ) { + deferred.resolve( item ); + return; + } + translateTextWithTagOffsets( + sourceLang, + targetLang, + item.item.getPlainText(), + item.item.getTagOffsets() + ).then( function ( translated ) { + var newTextBlock; + try { + newTextBlock = item.item.translateTags( + translated.text, translated.rangeMappings + ); + deferred.resolve( { + type: 'textblock', + item: newTextBlock + } ); + } catch ( ex ) { + deferred.reject( ex ); } - translateText( sourceLang, targetLang, item.item.getPlainText() ).then( function ( translated ) { - var newTextBlock; - try { - newTextBlock = item.item.translateAnnotations( - translated.text, translated.rangeMappings - ); - deferred.resolve( { - type: 'textblock', - item: newTextBlock - } ); - } catch ( ex ) { - deferred.reject( ex ); - } - } ); - }( Q.defer(), sourceDoc.items[ i ] ) ); + }, function ( error ) { + deferred.reject( error ); + } ); + } + for ( i = 0, len = sourceDoc.items.length; i < len; i++ ) { + translateItemDeferred( Q.defer(), sourceDoc.items[ i ] ); } deferred = Q.defer(); Q.all( itemPromises ).spread( function () { targetDoc.items = Array.prototype.slice.call( arguments, 0 ); - return deferred.resolve( targetDoc.getHtml() ); + deferred.resolve( targetDoc.getHtml() ); + }, function ( error ) { + deferred.reject( error ); } ); return deferred.promise; } module.exports = { translate: translate, - translateHtml: translateHtml, - translateText: translateText + translateLinesApertium: translateLinesApertium, + getTokens: getTokens }; diff --git a/tests/apertium/Apertium.test.js b/tests/apertium/Apertium.test.js deleted file mode 100644 index ff9ae93..0000000 --- a/tests/apertium/Apertium.test.js +++ /dev/null @@ -1,8 +0,0 @@ -var caHtml, Apertium = require( '../../mt/Apertium' ); - -caHtml = '<p>un dos <b><a href="3">tres</a> quatre <i><a href="5">cinc</a> sis set</i> vuit nou</b> deu</p>.'; - -Apertium.translate( 'ca', 'es', caHtml ).then( function ( esHtml ) { - console.log( 'caHtml:', caHtml ); - console.log( 'esHtml:', esHtml ); -} ); diff --git a/tests/index.js b/tests/index.js index 17ee135..ddf10d2 100644 --- a/tests/index.js +++ b/tests/index.js @@ -3,6 +3,7 @@ tests = [ './tests/segmentation/CXSegmenter.test.js', './tests/lineardoc/LinearDoc.test.js', + './tests/mt/Apertium.test.js', './tests/dictionary/Dictionary.test.js' ]; diff --git a/tests/lineardoc/LinearDoc.test.js b/tests/lineardoc/LinearDoc.test.js index 7b09d22..745fb36 100644 --- a/tests/lineardoc/LinearDoc.test.js +++ b/tests/lineardoc/LinearDoc.test.js @@ -32,7 +32,7 @@ } } ); -QUnit.test( 'LinearDoc translateAnnotations', function ( assert ) { +QUnit.test( 'LinearDoc translatetags', function ( assert ) { var parser, textBlock1, textBlock2, i, len, test, doc; QUnit.expect( 2 * transTests.length ); @@ -49,7 +49,7 @@ test.source, 'Reconstructed source HTML' ); - textBlock2 = textBlock1.translateAnnotations( + textBlock2 = textBlock1.translateTags( test.targetText, test.rangeMappings ); diff --git a/tests/lineardoc/translate.test.json b/tests/lineardoc/translate.test.json index da48d1c..0508548 100644 --- a/tests/lineardoc/translate.test.json +++ b/tests/lineardoc/translate.test.json @@ -46,6 +46,23 @@ "expect": "S \n" }, { + "source": "Un cotxe blau.", + "targetText": "A blue car.", + "rangeMappings": [], + "expect": "A blue car." +}, +{ + "source": "Un <a href=\"x\">cotxe</a> blau.", + "targetText": "A blue car.", + "rangeMappings": [ + { + "source": { "start": 3, "length": 5 }, + "target": { "start": 7, "length": 3 } + } + ], + "expect": "A blue <a href=\"x\">car</a>." +}, +{ "source": "<b>a1 a2</b> <u>b1 b2</u> <i>c1 c2</i>", "targetText": "C1 A1 B1 B2 A2 C2", "rangeMappings": [ diff --git a/tests/mt/Apertium.test.js b/tests/mt/Apertium.test.js new file mode 100644 index 0000000..7d2c5a9 --- /dev/null +++ b/tests/mt/Apertium.test.js @@ -0,0 +1,129 @@ +QUnit.module( 'Apertium' ); + +var tests, + Q = require( 'q' ); + +// In each case, below, just "source" and "textTranslations" should be sufficient for linearDoc +// to derive "target". // The plaintext strings in "textTranslations" are real Apertium output. +// They are pre-cached (Apertium is not actually called during the test), so that: +// 1. Unit testing does not depend on Apertium +// 2. Tests will not break if an Apertium upgrade changes some translations +tests = [ + { + title: 'All caps words', + source: '<p>A <b>Japanese</b> <i>BBC</i> article</p>', + target: '<p>Un artículo de BBC <b>japonés</b></p>', + textTranslations: { + 'A Japanese BBC article': 'Un artículo de BBC japonés', + 'A JAPANESE BBC article': 'Un artículo de BBC JAPONÉS' + } + }, + { + title: 'Title caps one-to-many', + source: '<div>A <b>modern</b> Britain.</div>', + target: '<div>Una Gran Bretaña <b>moderna</b>.</div>', + textTranslations: { + 'A modern Britain.': 'Una Gran Bretaña moderna.', + 'A MODERN Britain.': 'Una Gran Bretaña MODERNA.', + 'A modern BRITAIN.': 'Una GRAN BRETAÑA moderna.' + } + }, + { + title: 'Reordering with nested tags', + source: '<p>The <b>big <i>red</i></b> dog</p>', + target: '<p>El perro <b><i>rojo</i></b> <b>grande</b></p>', + textTranslations: { + 'The big red dog': 'El perro rojo grande', + 'The BIG red dog': 'El perro rojo GRANDE', + 'The big RED dog': 'El perro ROJO grande' + } + }, + { + title: 'Many-to-one with nested tags', + source: '<p>He said "<i>I tile <a href="x">bathrooms</a>.</i>"</p>', + target: '<p>Diga que "<i>enladrillo</i> <i><a href="x">baños</a></i>."</p>', + textTranslations: { + 'He said "I tile bathrooms."': 'Diga que "enladrillo baños."', + 'He said "I TILE bathrooms."': 'Diga que "ENLADRILLO baños."', + 'He said "I tile BATHROOMS."': 'Diga que "enladrillo BAÑOS."' + } + }, + { + title: 'Reordering at either ends of a tag', + source: '<p>The <b>big red</b> dog</p>', + target: '<p>El perro <b>rojo grande</b></p>', + textTranslations: { + 'The big red dog': 'El perro rojo grande', + 'The BIG RED dog': 'El perro ROJO GRANDE' + } + }, + { + title: 'Identical tags separated by whitespace', + source: '<p>The <b>big</b> <b>red</b> dog</p>', + target: '<p>El perro <b>rojo</b> <b>grande</b></p>', + textTranslations: { + 'The big red dog': 'El perro rojo grande', + 'The BIG red dog': 'El perro rojo GRANDE', + 'The big RED dog': 'El perro ROJO grande' + } + }, + { + title: 'Non-identical links separated by whitespace', + source: '<p>The <a href="1">big</a> <a href="2">red</a> dog</p>', + target: '<p>El perro <a href="2">rojo</a> <a href="1">grande</a></p>', + textTranslations: { + 'The big red dog': 'El perro rojo grande', + 'The BIG red dog': 'El perro rojo GRANDE', + 'The big RED dog': 'El perro ROJO grande' + } + } +]; + +QUnit.test( 'Apertium wrapper tests', function ( assert ) { + var textTranslations; + + // Fake the actual Apertium call + CX.Apertium.translateLinesApertium = function ( sourceLang, targetLang, sourceLines ) { + var deferred = Q.defer(); + setTimeout( function () { + var targetLines; + try { + sourceLines.map( function ( line ) { + return textTranslations[ line ] || 'X' + line + 'X'; + } ); + deferred.resolve( targetLines ); + } catch ( error ) { + deferred.reject( error ); + } + } ); + return deferred.promise; + }; + + QUnit.expect( tests.length ); + + function resumeTests( i ) { + var test; + if ( i >= tests.length ) { + return; + } + test = tests[ i ]; + textTranslations = test.textTranslations; + + QUnit.stop(); + + CX.Apertium.translate( 'xx', 'yy', test.source ).then( function ( target ) { + assert.strictEqual( + target, + test.target, + test.title + ); + QUnit.start(); + resumeTests( i + 1 ); + }, function ( error ) { + assert.ok( false, test.title + ': ' + error ); + QUnit.start(); + resumeTests( i + 1 ); + } ); + } + resumeTests( 0 ); +} ); -- To view, visit https://gerrit.wikimedia.org/r/157141 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I6d1483e69aae979a684728f7724f6874194eee11 Gerrit-PatchSet: 12 Gerrit-Project: mediawiki/services/cxserver Gerrit-Branch: master Gerrit-Owner: Divec <[email protected]> Gerrit-Reviewer: Divec <[email protected]> Gerrit-Reviewer: Santhosh <[email protected]> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
