jenkins-bot has submitted this change and it was merged.
Change subject: translateAnnotations with partial mapping
......................................................................
translateAnnotations with partial mapping
LinearDoc:
* Allow partial rangeMappings for translateAnnotations
* Fallback to plaintext where range info is incomplete
* Fallback to segment end placement for references
* Inline span['data-mw'] (like img) if it only contains whitespace
LinearDoc.test.js, translate.test.json
* Test translateAnnotations with plaintext translation and range mapping
Change-Id: I960dbe6155a5ea859b716e4a8748002a513618a6
---
M lineardoc/LinearDoc.js
M tests/lineardoc/LinearDoc.test.js
A tests/lineardoc/translate.test.json
3 files changed, 235 insertions(+), 13 deletions(-)
Approvals:
Santhosh: Looks good to me, approved
jenkins-bot: Verified
diff --git a/lineardoc/LinearDoc.js b/lineardoc/LinearDoc.js
index 92e8549..58a0cdd 100644
--- a/lineardoc/LinearDoc.js
+++ b/lineardoc/LinearDoc.js
@@ -316,20 +316,137 @@
* @returns {TextBlock} Translated textblock with annotations applied
*/
TextBlock.prototype.translateAnnotations = function ( targetText,
rangeMappings ) {
- var i, len, rangeMapping, oldTextChunk, newText,
- newTextChunks = [];
+ var i, iLen, j, rangeMapping, sourceTextChunk, text, pos, textChunk,
offset,
+ sourceRangeEnd, targetRangeEnd, tail, tailSpace,
+ // map of { offset: x, textChunks: [...] }
+ emptyTextChunks = {},
+ emptyTextChunkOffsets = [],
+ // list of { start: x, length: x, textChunk: x }
+ textChunks = [];
- for ( i = 0, len = rangeMappings.length; i < len; i++ ) {
- rangeMapping = rangeMappings[ i ];
- oldTextChunk = this.getTextChunkAt( rangeMapping.source.start );
- newText = targetText.substr( rangeMapping.target.start,
rangeMapping.target.length );
- newTextChunks.push( new TextChunk(
- newText,
- oldTextChunk.tags,
- oldTextChunk.inlineContent
- ) );
+ function pushEmptyTextChunks( offset, chunks ) {
+ var c, cLen;
+ for ( c = 0, cLen = chunks.length; c < cLen; c++ ) {
+ textChunks.push( {
+ start: offset,
+ length: 0,
+ textChunk: chunks[ c ]
+ } );
+ }
}
- return new TextBlock( newTextChunks );
+
+ // Create map of empty text chunks, by offset
+ for ( i = 0, iLen = this.textChunks.length; i < iLen; i++ ) {
+ textChunk = this.textChunks[ i ];
+ offset = this.startOffsets[ i ];
+ if ( textChunk.text.length > 0 ) {
+ continue;
+ }
+ if ( !emptyTextChunks[ offset ] ) {
+ emptyTextChunks[ offset ] = [];
+ }
+ emptyTextChunks[ offset ].push( textChunk );
+ }
+ for ( offset in emptyTextChunks ) {
+ emptyTextChunkOffsets.push( offset );
+ }
+ emptyTextChunkOffsets.sort( function ( a, b ) { return a - b; } );
+
+ for ( i = 0, iLen = rangeMappings.length; i < iLen; i++ ) {
+ // Copy annotations from source text start offset
+ rangeMapping = rangeMappings[ i ];
+ sourceRangeEnd = rangeMapping.source.start +
rangeMapping.source.length;
+ targetRangeEnd = rangeMapping.target.start +
rangeMapping.target.length;
+ sourceTextChunk = this.getTextChunkAt(
rangeMapping.source.start );
+ text = targetText.substr( rangeMapping.target.start,
rangeMapping.target.length );
+ textChunks.push( {
+ start: rangeMapping.target.start,
+ length: rangeMapping.target.length,
+ textChunk: new TextChunk(
+ text,
+ sourceTextChunk.tags,
+ sourceTextChunk.inlineContent
+ )
+ } );
+
+ // Empty source text chunks will not be represented in the
target plaintext
+ // (because they have no plaintext representation). Therefore
we must clone each
+ // one manually into the target rich text.
+
+ // Iterate through all remaining emptyTextChunks
+ for ( j = 0; j < emptyTextChunkOffsets.length; j++ ) {
+ offset = emptyTextChunkOffsets[ j ];
+ // Check whether chunk is in range
+ if ( offset < rangeMapping.source.start || offset >
sourceRangeEnd ) {
+ continue;
+ }
+ // Push chunk into target text at the current point
+ pushEmptyTextChunks( targetRangeEnd, emptyTextChunks[
offset ] );
+ // Remove chunk from remaining list
+ delete emptyTextChunks[ offset ];
+ emptyTextChunkOffsets.splice( j, 1 );
+ // Decrement pointer to match removal
+ j--;
+ }
+ }
+ // Sort by start position
+ textChunks.sort( function ( textChunk1, textChunk2 ) {
+ return textChunk1.start - textChunk2.start;
+ } );
+ // Fill in any textChunk gaps with unannotated text
+ pos = 0;
+ for ( i = 0, iLen = textChunks.length; i < iLen; i++ ) {
+ textChunk = textChunks[ i ];
+ if ( textChunk.start < pos ) {
+ throw new Error( 'Overlappping chunks at pos=' + pos +
', i=' + i );
+ } else if ( textChunk.start > pos ) {
+ // Unmapped chunk: insert plaintext and adjust offset
+ textChunks.splice( i, 0, {
+ start: pos,
+ length: textChunk.start - pos,
+ textChunk: new TextChunk(
+ targetText.substr( pos, textChunk.start
- pos ),
+ []
+ )
+ } );
+ i++;
+ iLen++;
+ }
+ pos = textChunk.start + textChunk.length;
+ }
+
+ // Get trailing text and trailing whitespace
+ tail = targetText.substr( pos );
+ tailSpace = tail.match( /\s*$/ )[ 0 ];
+ if ( tailSpace ) {
+ tail = tail.substr( 0, tail.length - tailSpace.length );
+ }
+
+ if ( tail ) {
+ // Append tail as unannotated text
+ textChunks.push( {
+ start: pos,
+ length: tail.length,
+ textChunk: new TextChunk( tail, [] )
+ } );
+ pos += tail.length;
+ }
+
+ // Copy any remaining textChunks that have no text
+ for ( i = 0, iLen = emptyTextChunkOffsets.length; i < iLen; i++ ) {
+ offset = emptyTextChunkOffsets[ i ];
+ pushEmptyTextChunks( pos, emptyTextChunks[ offset ] );
+ }
+ if ( tailSpace ) {
+ // Append tailSpace as unannotated text
+ textChunks.push( {
+ start: pos,
+ length: tailSpace.length,
+ textChunk: new TextChunk( tailSpace, [] )
+ } );
+ pos += tail.length;
+ }
+ return new TextBlock( textChunks.map( function ( x ) { return
x.textChunk; } ) );
};
/**
diff --git a/tests/lineardoc/LinearDoc.test.js
b/tests/lineardoc/LinearDoc.test.js
index af13187..7b09d22 100644
--- a/tests/lineardoc/LinearDoc.test.js
+++ b/tests/lineardoc/LinearDoc.test.js
@@ -1,6 +1,7 @@
QUnit.module( 'LinearDoc' );
-var fs = require( 'fs' );
+var fs = require( 'fs' ),
+ transTests = require( __dirname + '/translate.test.json' );
QUnit.test( 'LinearDoc tests', function ( assert ) {
var parser, testXhtmlFile, resultXmlFile, resultXhtmlFile, testXhtml,
resultXml,
@@ -30,3 +31,32 @@
);
}
} );
+
+QUnit.test( 'LinearDoc translateAnnotations', function ( assert ) {
+ var parser, textBlock1, textBlock2, i, len, test, doc;
+
+ QUnit.expect( 2 * transTests.length );
+
+ for ( i = 0, len = transTests.length; i < len; i++ ) {
+ test = transTests[ i ];
+ parser = new CX.LinearDoc.Parser();
+ parser.init();
+ parser.write( '<div>' + test.source + '</div>' );
+ doc = parser.builder.doc;
+ textBlock1 = parser.builder.doc.items[ 1 ].item;
+ assert.strictEqual(
+ textBlock1.getHtml(),
+ test.source,
+ 'Reconstructed source HTML'
+ );
+ textBlock2 = textBlock1.translateAnnotations(
+ test.targetText,
+ test.rangeMappings
+ );
+ assert.strictEqual(
+ textBlock2.getHtml(),
+ test.expect,
+ 'Translated HTML'
+ );
+ }
+} );
diff --git a/tests/lineardoc/translate.test.json
b/tests/lineardoc/translate.test.json
new file mode 100644
index 0000000..da48d1c
--- /dev/null
+++ b/tests/lineardoc/translate.test.json
@@ -0,0 +1,75 @@
+[
+{
+ "source": "A <b>big<span typeof=\"mw:Extension/ref\">[1]</span>
<i>brown</i><span typeof=\"mw:Extension/ref\">[2]</span> </b><u>dog</u> from
<img /> the <a href=\"GB\">United Kingdom</a><span
typeof=\"mw:Extension/ref\">[3]</span>. ",
+ "targetText": "The hound (enormous and brown) from Great Britain. ",
+ "rangeMappings": [
+ {
+ "source": { "start": 12, "length": 3 },
+ "target": { "start": 4, "length": 5 }
+ },
+ {
+ "source": { "start": 2, "length": 9 },
+ "target": { "start": 11, "length": 18 }
+ },
+ {
+ "source": { "start": 27, "length": 14 },
+ "target": { "start": 36, "length": 13 }
+ }
+ ],
+ "expect": "The <u>hound</u> (<b>enormous and brown<span
typeof=\"mw:Extension/ref\">[1]</span><span
typeof=\"mw:Extension/ref\">[2]</span></b>) from <a href=\"GB\">Great
Britain</a><span typeof=\"mw:Extension/ref\">[3]</span>.<img /> "
+},
+{
+ "source": "x <span data-mw=\"foo\" />y <span data-mw=\"bar\" />z. ",
+ "targetText": "Z X Y. ",
+ "rangeMappings": [
+ {
+ "source": { "start": 0, "length": 3 },
+ "target": { "start": 2, "length": 3 }
+ },
+ {
+ "source": { "start": 5, "length": 1 },
+ "target": { "start": 0, "length": 1 }
+ }
+ ],
+ "expect": "Z X Y<span data-mw=\"foo\" />.<span data-mw=\"bar\" /> "
+},
+{
+ "source": "<span data-mw=\"x\">\n</span>",
+ "targetText": "",
+ "rangeMappings": [],
+ "expect": "<span data-mw=\"x\">\n</span>"
+},
+{
+ "source": "s <span>\n</span>",
+ "targetText": "S \n",
+ "rangeMappings": [],
+ "expect": "S \n"
+},
+{
+ "source": "<b>a1 a2</b> <u>b1 b2</u> <i>c1 c2</i>",
+ "targetText": "C1 A1 B1 B2 A2 C2",
+ "rangeMappings": [
+ {
+ "source": { "start": 0, "length": 5 },
+ "target": { "start": 3, "length": 2 }
+ },
+ {
+ "source": { "start": 0, "length": 5 },
+ "target": { "start": 12, "length": 2 }
+ },
+ {
+ "source": { "start": 6, "length": 5 },
+ "target": { "start": 6, "length": 5 }
+ },
+ {
+ "source": { "start": 12, "length": 5 },
+ "target": { "start": 0, "length": 2 }
+ },
+ {
+ "source": { "start": 12, "length": 5 },
+ "target": { "start": 15, "length": 2 }
+ }
+ ],
+ "expect": "<i>C1</i> <b>A1</b> <u>B1 B2</u> <b>A2</b> <i>C2</i>"
+}
+]
--
To view, visit https://gerrit.wikimedia.org/r/156518
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I960dbe6155a5ea859b716e4a8748002a513618a6
Gerrit-PatchSet: 8
Gerrit-Project: mediawiki/services/cxserver
Gerrit-Branch: master
Gerrit-Owner: Divec <[email protected]>
Gerrit-Reviewer: Amire80 <[email protected]>
Gerrit-Reviewer: Divec <[email protected]>
Gerrit-Reviewer: Santhosh <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
