jenkins-bot has submitted this change and it was merged.
Change subject: Use LinearDoc for segmentation
......................................................................
Use LinearDoc for segmentation
Use LinearDoc to allow apply segmentation at a structural level.
Ensure valid, well-formed HTML output.
Include inter-segment whitespace with the previous segment.
Treat references as zero-width annotations; do not segment inside them.
Do not give link IDs to links inside a reference.
LinearDoc.js
* Add segment IDs to HTML output
* Add link IDs to HTML output
* Output attributes in lexicographic order
* Add Normalizer class for test data
CXSegmenter.js
* Rewrite completely, using LinearDoc.js
Segmenter{Default,En,Hi}.js
* Basic plaintext segmenters that work well with LinearDoc segmentation
CXSegmenter.test.js
* Normalize expected/actual output before comparison
tests/segmentation/data/result*.html
* Rewrite for LinearDoc segmentation (remove some hardcoded limitations)
* Add some new tests
* Disable one test ('Mr. D. Jones'); add a new one ('The UK. The US')
ContentTranslationService.js
* Pass the source language
Change-Id: Iee6bfa499c9f23269a290df5de5e027e48746971
---
M ContentTranslationService.js
M lineardoc/LinearDoc.js
M segmentation/CXSegmenter.js
A segmentation/SegmenterDefault.js
A segmentation/SegmenterEn.js
A segmentation/SegmenterHi.js
M tests/segmentation/CXSegmenter.test.js
M tests/segmentation/SegmentationTests.json
M tests/segmentation/data/result-10.html
M tests/segmentation/data/result-11.html
M tests/segmentation/data/result-12.html
M tests/segmentation/data/result-13.html
M tests/segmentation/data/result-15.html
A tests/segmentation/data/result-16.html
A tests/segmentation/data/result-17.html
M tests/segmentation/data/result-7.html
M tests/segmentation/data/result-8.html
M tests/segmentation/data/result-9.html
M tests/segmentation/data/result-debian-1.html
M tests/segmentation/data/result-ends-with-references-missing-letters.html
A tests/segmentation/data/test-16.html
A tests/segmentation/data/test-17.html
M tests/segmentation/data/test-7.html
23 files changed, 432 insertions(+), 147 deletions(-)
Approvals:
Santhosh: Looks good to me, approved
jenkins-bot: Verified
diff --git a/ContentTranslationService.js b/ContentTranslationService.js
index 3d6e0c7..017f868 100644
--- a/ContentTranslationService.js
+++ b/ContentTranslationService.js
@@ -67,7 +67,7 @@
var segmenter, segmentedContent;
try {
logger.debug( 'Page fetched' );
- segmenter = new CXSegmenter( data );
+ segmenter = new CXSegmenter( data, sourceLanguage );
segmenter.segment();
segmentedContent = segmenter.getSegmentedContent();
} catch ( error ) {
diff --git a/lineardoc/LinearDoc.js b/lineardoc/LinearDoc.js
index 54aba8f..4337314 100644
--- a/lineardoc/LinearDoc.js
+++ b/lineardoc/LinearDoc.js
@@ -59,9 +59,15 @@
* @return {string} Html representation of open tag
*/
function getOpenTagHtml( tag ) {
- var attr, html;
+ var html, attributes, attr, i, len;
html = [ '<' + esc( tag.name ) ];
+ attributes = [];
for ( attr in tag.attributes ) {
+ attributes.push( attr );
+ }
+ attributes.sort();
+ for ( i = 0, len = attributes.length; i < len; i++ ) {
+ attr = attributes[i];
html.push( ' ' + esc( attr ) + '="' + escAttr( tag.attributes[
attr ] ) + '"' );
}
if ( tag.isSelfClosing ) {
@@ -69,6 +75,20 @@
}
html.push( '>' );
return html.join( '' );
+}
+
+/**
+ * Clone a SAX open tag
+ * @private
+ * @param {Object} tag Tag to clone
+ * @return {Object} Cloned tag
+ */
+function cloneOpenTag( tag ) {
+ var attr, newTag = { name: tag.name, attributes: {} };
+ for ( attr in tag.attributes ) {
+ newTag.attributes[attr] = tag.attributes[attr];
+ }
+ return newTag;
}
/**
@@ -119,7 +139,13 @@
* @return {boolean} Whether the tag is a mediawiki reference span
*/
function isReference( tag ) {
- return tag.name === 'span' && tag.attributes.typeof ===
'mw:Extension/ref';
+ if ( tag.name === 'span' && tag.attributes.typeof ===
'mw:Extension/ref' ) {
+ return true;
+ } else if ( tag.name === 'span' && tag.attributes.class === 'reference'
) {
+ // TODO: This is in the tests, but is it correct behaviour?
+ return true;
+ }
+ return false;
}
/**
@@ -173,6 +199,55 @@
// non-annotation inline tags
'img', 'br'
] ) );
+
+/**
+ * Find the boundaries that lie in each chunk
+ *
+ * Boundaries lying between chunks lie in the latest chunk possible.
+ * Boundaries at the start of the first chunk, or the end of the last, are not
included.
+ * Therefore zero-width chunks never have any boundaries
+ *
+ * @function
+ * @param {number[]} boundaries Boundary offsets
+ * @param chunks Chunks to which the boundaries apply
+ * @param {Function} getLength Function returning the length of a chunk
+ * @returns {Object} Array of {chunk: ch, boundaries: [...]}
+ */
+function getChunkBoundaryGroups( boundaries, chunks, getLength ) {
+ var i, len, groupBoundaries, chunk, chunkLength, boundary,
+ groups = [],
+ offset = 0,
+ boundaryPtr = 0;
+
+ // Get boundaries in order, disregarding the start of the first chunk
+ boundaries = boundaries.slice();
+ boundaries.sort( function ( a, b ) { return a - b; } );
+ while ( boundaries[boundaryPtr] === 0 ) {
+ boundaryPtr++;
+ }
+ for( i = 0, len = chunks.length; i < len; i++ ) {
+ groupBoundaries = [];
+ chunk = chunks[i];
+ chunkLength = getLength( chunk );
+ while ( true ) {
+ boundary = boundaries[boundaryPtr];
+ if ( boundary === undefined || boundary > offset +
chunkLength - 1 ) {
+ // beyond the interior of this chunk
+ break;
+ }
+ // inside the interior of this chunk
+ groupBoundaries.push( boundary );
+ boundaryPtr++;
+ }
+ offset += chunkLength;
+ groups.push( {
+ chunk: chunk,
+ boundaries: groupBoundaries
+ } );
+ // Continue even if past boundaries: need to add remaining
chunks
+ }
+ return groups;
+}
/**
* A chunk of uniformly-annotated inline text
@@ -311,77 +386,104 @@
}
/**
+ * Set link IDs in-place on text chunks
+ *
+ * @private
+ * @param {TextChunk[]} textChunks Consecutive text chunks
+ * @param {Function} getNextId function accepting 'link' and returning next ID
+ */
+function setLinkIdsInPlace( textChunks, getNextId ) {
+ var i, iLen, j, jLen, tags, tag, href;
+ for ( i = 0, iLen = textChunks.length; i < iLen; i++ ) {
+ tags = textChunks[ i ].tags;
+ for ( j = 0, jLen = tags.length; j < jLen; j++ ) {
+ tag = tags[ j ];
+ if (
+ tag.name === 'a' &&
+ tag.attributes.href !== undefined &&
+ tag.attributes[ 'data-linkid' ] === undefined
+ ) {
+ // Hack: copy href, then remove it, then re-add
it, so that
+ // attributes appear in alphabetical order (ugh)
+ href = tag.attributes.href;
+ delete tag.attributes.href;
+ tag.attributes.class = 'cx-link';
+ tag.attributes[ 'data-linkid' ] = getNextId(
'link' );
+ tag.attributes.href = href;
+ }
+ }
+ }
+}
+
+/**
* Segment the text block into sentences
* @method
* @param {Function} getBoundaries Function taking plaintext, returning offset
array
+ * @param {Function} getNextId Function taking 'segment'|'link', returning
next ID
* @return {TextBlock} Segmented version, with added span tags
*/
-TextBlock.prototype.segment = function ( getBoundaries ) {
- var i, len, textChunk, boundary, relOffset,
- allTextChunks = [],
- currentTextChunks = [],
- charCount = 0,
- boundaries = getBoundaries( this.getPlainText() ),
- bPtr = 0,
- segId = 1;
+TextBlock.prototype.segment = function ( getBoundaries, getNextId ) {
+ var allTextChunks, currentTextChunks, groups, i, iLen, group, offset,
textChunk, j, jLen,
+ leftPart, rightPart, boundaries, relOffset;
+ // Setup: currentTextChunks for current segment, and allTextChunks for
all segments
+ allTextChunks = [];
+ currentTextChunks = [];
function flushChunks() {
- if ( currentTextChunks.length > 0 ) {
- allTextChunks.push.apply( allTextChunks, addCommonTag(
- currentTextChunks, {
- name: 'span',
- attributes: {
- class: 'seg s' + segId++
- }
- }
- ) );
- currentTextChunks = [];
+ var modifiedTextChunks;
+ if ( currentTextChunks.length === 0 ) {
+ return;
}
+ modifiedTextChunks = addCommonTag(
+ currentTextChunks,
+ {
+ name: 'span',
+ attributes: {
+ 'class': 'cx-segment',
+ 'data-segmentid': getNextId( 'segment' )
+ }
+ }
+ );
+ setLinkIdsInPlace( modifiedTextChunks, getNextId );
+ allTextChunks.push.apply( allTextChunks, modifiedTextChunks );
+ currentTextChunks = [];
}
- for ( i = 0, len = this.textChunks.length; i < len; i++ ) {
- textChunk = this.textChunks[ i ];
- // Move boundary pointer to the boundary after the start of
this chunk
- // (or beyond the end of the array if there is no such boundary)
- // Notice that if the chunk is zero-length, bPtr may already be
exactly
- // at the end of the chunk, in which case bPtr won't move
- while ( bPtr < boundaries.length && boundaries[ bPtr ] <
charCount ) {
- bPtr++;
- }
- boundary = boundaries[ bPtr ];
+ // for each chunk, split at any boundaries that occur inside the chunk
+ groups = getChunkBoundaryGroups(
+ getBoundaries( this.getPlainText() ),
+ this.textChunks,
+ function ( textChunk ) { return textChunk.text.length; }
+ );
- // Get offset relative to the start of this text chunk
- relOffset = ( boundary === undefined ) ? undefined : boundary -
charCount;
- if ( relOffset === 0 ) {
- // Boundary exactly at start of text
- if ( textChunk.text.length === 0 ) {
- // Zero-width: chunk lies before segment
boundary
- // Don't flush chunks yet: another zero-width
chunk may come
- currentTextChunks.push( textChunk );
- } else {
- // Non-zero width: chunk lies after segment
boundary, so flush
+ offset = 0;
+ for ( i = 0, iLen = groups.length; i < iLen; i++ ) {
+ group = groups[i];
+ textChunk = group.chunk;
+ boundaries = group.boundaries;
+ for ( j = 0, jLen = boundaries.length; j < jLen; j++ ) {
+ relOffset = boundaries[j] - offset;
+ if ( relOffset === 0 ) {
flushChunks();
- currentTextChunks.push( textChunk );
+ } else {
+ leftPart = new TextChunk(
+ textChunk.text.substring( 0, relOffset
),
+ textChunk.tags.slice()
+ );
+ rightPart = new TextChunk(
+ textChunk.text.substring( relOffset ),
+ textChunk.tags.slice(),
+ textChunk.inlineElement
+ );
+ currentTextChunks.push( leftPart );
+ offset += relOffset;
+ flushChunks();
+ textChunk = rightPart;
}
- } else if ( relOffset < textChunk.text.length ) {
- // Boundary strictly inside the text: split chunk
- // Add pre-split chunk, then flush block
- currentTextChunks.push( new TextChunk(
- textChunk.text.substring( 0, relOffset ),
- textChunk.tags.slice()
- ) );
- flushChunks();
- // Add post-split chunk, including ref if any
- currentTextChunks.push( new TextChunk(
- textChunk.text.substring( relOffset ),
- textChunk.tags.slice(),
- textChunk.inlineElement
- ) );
- } else {
- // No boundary, or boundary after chunk: add whole chunk
- currentTextChunks.push( textChunk );
}
- charCount += textChunk.text.length;
+ // Even if the textChunk is zero-width, it may have references
+ currentTextChunks.push( textChunk );
+ offset += textChunk.text.length;
}
flushChunks();
return new TextBlock( allTextChunks );
@@ -460,15 +562,33 @@
* @return {Doc} Segmented version of document TODO: warning: *shallow copied*.
*/
Doc.prototype.segment = function ( getBoundaries ) {
- var i, len, item, textBlock,
- newDoc = new Doc();
+ var i, len, item, tag, textBlock,
+ newDoc = new Doc(),
+ nextId = 0;
+
+ // TODO: return different counters depending on type
+ function getNextId( type ) {
+ if ( type === 'segment' || type === 'link' || type === 'block'
) {
+ return '' + nextId++;
+ } else {
+ throw new Error( 'Unknown ID type: ' + type );
+ }
+ }
+
for ( i = 0, len = this.items.length; i < len; i++ ) {
item = this.items[ i ];
- if ( this.items[ i ].type !== 'textblock' ) {
+ if ( this.items[ i ].type === 'open' ) {
+ tag = cloneOpenTag( item.item );
+ tag.attributes.id = getNextId( 'block' );
+ newDoc.addItem( item.type, tag );
+ } else if ( this.items[ i ].type !== 'textblock' ) {
newDoc.addItem( item.type, item.item );
} else {
textBlock = item.item;
- newDoc.addItem( 'textblock', textBlock.segment(
getBoundaries ) );
+ newDoc.addItem(
+ 'textblock',
+ textBlock.segment( getBoundaries, getNextId )
+ );
}
}
return newDoc;
@@ -733,11 +853,48 @@
}
};
+/**
+ * Parser to normalize XML
+ * @class
+ * @constructor
+ */
+function Normalizer() {
+ SAXParser.call( this, false, { lowercase: true } );
+}
+util.inherits( Normalizer, SAXParser );
+
+Normalizer.prototype.init = function () {
+ this.doc = [];
+ this.tags = [];
+};
+
+Normalizer.prototype.onopentag = function ( tag ) {
+ this.tags.push( tag );
+ this.doc.push( getOpenTagHtml( tag ) );
+};
+
+Normalizer.prototype.onclosetag = function ( tagName ) {
+ var tag = this.tags.pop();
+ if ( tag.name !== tagName ) {
+ throw new Error( 'Unmatched tags: ' + tag.name + ' !== ' +
tagName );
+ }
+ this.doc.push( getCloseTagHtml( tag ) );
+};
+
+Normalizer.prototype.ontext = function ( text ) {
+ this.doc.push( esc( text ) );
+};
+
+Normalizer.prototype.getHtml = function () {
+ return this.doc.join( '' );
+};
+
module.exports = {
findAll: findAll,
Doc: Doc,
TextBlock: TextBlock,
TextChunk: TextChunk,
Builder: Builder,
- Parser: Parser
+ Parser: Parser,
+ Normalizer: Normalizer
};
diff --git a/segmentation/CXSegmenter.js b/segmentation/CXSegmenter.js
index 3ba9e5e..9155ab1 100644
--- a/segmentation/CXSegmenter.js
+++ b/segmentation/CXSegmenter.js
@@ -9,52 +9,40 @@
'use strict';
-var CXParserFactory = require( __dirname + '/CXParserFactory.js'
).CXParserFactory,
- $ = require( 'jquery' );
+var LinearDoc = require( '../lineardoc/LinearDoc' ),
+ logger = require( __dirname + '/../utils/Logger.js' ),
+ getBoundariesDefault = require( './SegmenterDefault' ).getBoundaries,
+ getBoundariesEn = require( './SegmenterEn' ).getBoundaries,
+ getBoundariesHi = require( './SegmenterHi' ).getBoundaries;
+
+function getBoundaryFunction( language ) {
+ if ( language === 'en' ) {
+ return getBoundariesEn;
+ } else if ( language === 'hi' ) {
+ return getBoundariesHi;
+ } else {
+ logger.warn( 'Using fallback boundary function for language: '
+ JSON.stringify( language ) );
+ return getBoundariesDefault;
+ }
+}
function CXSegmenter( content, language ) {
+ this.parser = new LinearDoc.Parser();
+ this.parser.init();
+ this.getBoundaries = getBoundaryFunction( language );
this.content = content;
- this.segments = {};
- this.segmentedContent = null;
- this.links = {};
- this.parser = ( new CXParserFactory() ).getParser( language || 'en' );
+ this.originalDoc = null;
+ this.segmentedDoc = null;
}
CXSegmenter.prototype.segment = function () {
- this.parse();
- this.extractSegments();
-};
-
-CXSegmenter.prototype.parse = function () {
- this.parser.parse( this.content );
- this.links = this.parser.links;
- this.segmentedContent = this.parser.segmentedContent;
-};
-
-CXSegmenter.prototype.getLinks = function () {
- return this.links;
-};
-
-CXSegmenter.prototype.extractSegments = function () {
- var segmenter = this,
- $container = $( '<div>' ).html( this.segmentedContent );
-
- $container.find( '.cx-segment' ).each( function ( index, section ) {
- var $section = $( section ),
- segmentId = $section.data( 'segmentid' );
-
- segmenter.segments[segmentId] = {
- source: $section.html()
- };
- } );
-};
-
-CXSegmenter.prototype.getSegments = function () {
- return this.segments;
+ this.parser.write( this.content );
+ this.originalDoc = this.parser.builder.doc;
+ this.segmentedDoc = this.originalDoc.segment( this.getBoundaries );
};
CXSegmenter.prototype.getSegmentedContent = function () {
- return this.segmentedContent;
+ return this.segmentedDoc.getHtml();
};
module.exports.CXSegmenter = CXSegmenter;
diff --git a/segmentation/SegmenterDefault.js b/segmentation/SegmenterDefault.js
new file mode 100644
index 0000000..782ce75
--- /dev/null
+++ b/segmentation/SegmenterDefault.js
@@ -0,0 +1,37 @@
+var findAll = require( '../lineardoc/LinearDoc' ).findAll;
+
+/**
+ * Test a possible sentence boundary match
+ *
+ * @param {string} text The plaintext to segment
+ * @param {Object} match The possible boundary match (returned by regex.exec)
+ * @return {number|null} The boundary offset, or null if not a sentence
boundary
+ */
+
+function findBoundary( text, match ) {
+ var tail = text.slice( match.index + 1, text.length );
+ // Trailing non-final punctuation: not a sentence boundary
+ if ( tail.match( /^[,;:]/ ) ) {
+ return null;
+ }
+ // Next word character is number or lower-case: not a sentence boundary
+ if ( tail.match( /^\W*[0-9a-z]/ ) ) {
+ return null;
+ }
+ // Include any closing punctuation and trailing space
+ return match.index + 1 + tail.match( /^['”"’]*\s*/ )[0].length;
+}
+
+/**
+ * Find English sentence boundaries
+ *
+ * @param {string} text The plaintext to segment
+ * @returns {number[]} Sentence boundary offsets
+ */
+function getBoundaries( text ) {
+ // Regex to find possible English sentence boundaries.
+ // Must not use a shared regex instance (re.lastIndex is used)
+ return findAll( text, /[.!?]/g, findBoundary );
+}
+
+module.exports = { getBoundaries: getBoundaries };
diff --git a/segmentation/SegmenterEn.js b/segmentation/SegmenterEn.js
new file mode 100644
index 0000000..800e6bd
--- /dev/null
+++ b/segmentation/SegmenterEn.js
@@ -0,0 +1,37 @@
+var findAll = require( '../lineardoc/LinearDoc' ).findAll;
+
+/**
+ * Test a possible English sentence boundary match
+ *
+ * @param {string} text The plaintext to segment
+ * @param {Object} match The possible boundary match (returned by regex.exec)
+ * @return {number|null} The boundary offset, or null if not a sentence
boundary
+ */
+
+function findBoundary( text, match ) {
+ var tail = text.slice( match.index + 1, text.length );
+ // Trailing non-final punctuation: not a sentence boundary
+ if ( tail.match( /^[,;:]/ ) ) {
+ return null;
+ }
+ // Next word character is number or lower-case: not a sentence boundary
+ if ( tail.match( /^\W*[0-9a-z]/ ) ) {
+ return null;
+ }
+ // Include any closing punctuation and trailing space
+ return match.index + 1 + tail.match( /^['”"’]*\s*/ )[0].length;
+}
+
+/**
+ * Find English sentence boundaries
+ *
+ * @param {string} text The plaintext to segment
+ * @returns {number[]} Sentence boundary offsets
+ */
+function getBoundaries( text ) {
+ // Regex to find possible English sentence boundaries.
+ // Must not use a shared regex instance (re.lastIndex is used)
+ return findAll( text, /[.!?]/g, findBoundary );
+}
+
+module.exports = { getBoundaries: getBoundaries };
diff --git a/segmentation/SegmenterHi.js b/segmentation/SegmenterHi.js
new file mode 100644
index 0000000..8b37888
--- /dev/null
+++ b/segmentation/SegmenterHi.js
@@ -0,0 +1,28 @@
+var findAll = require( '../lineardoc/LinearDoc' ).findAll;
+
+/**
+ * Test a possible Hindi sentence boundary match
+ *
+ * @param {string} text The plaintext to segment
+ * @param {Object} match The possible boundary match (returned by regex.exec)
+ * @return {number|null} The boundary offset, or null if not a sentence
boundary
+ */
+function findBoundary( text, match ) {
+ var tail = text.slice( match.index + 1, text.length );
+ // Include any trailing space
+ return match.index + 1 + tail.match( /^\s*/ )[0].length;
+}
+
+/**
+ * Find Hindi sentence boundaries
+ *
+ * @param {string} text The plaintext to segment
+ * @returns {number[]} Sentence boundary offsets
+ */
+function getBoundaries( text ) {
+ // Regex to find possible Hindi sentence boundaries.
+ // Must not use a shared regex instance (re.lastIndex is used)
+ return findAll( text, /[।!?]/g, findBoundary );
+}
+
+module.exports = { getBoundaries: getBoundaries };
diff --git a/tests/segmentation/CXSegmenter.test.js
b/tests/segmentation/CXSegmenter.test.js
index ee70d91..f1b2aea 100644
--- a/tests/segmentation/CXSegmenter.test.js
+++ b/tests/segmentation/CXSegmenter.test.js
@@ -1,27 +1,49 @@
QUnit.module( 'CXSegmenter' );
var fs = require( 'fs' );
+
+function normalize( html ) {
+ var normalizer = new CX.LinearDoc.Normalizer();
+ normalizer.init();
+ normalizer.write( html.replace( /(\r\n|\n|\t|\r)/gm, '' ) );
+ return normalizer.getHtml();
+}
+
QUnit.test( 'Segmentation tests', function ( assert ) {
- var i, len, lang, test, tests, segmenter, result,
- count = 0,testData,expectedResultData,
+ var i, len, lang, test, tests, segmenter, result, testData,
expectedResultData,
+ count = 0,
+ skipCount = 0,
allTests = require( './SegmentationTests.json' );
for ( lang in allTests ) {
- count += allTests[ lang ].length;
+ for ( i = 0, len = allTests[ lang ].length; i < len; i++ ) {
+ if ( allTests[ lang ][ i ].skip ) {
+ skipCount += 1;
+ } else {
+ count += 1;
+ }
+ }
}
QUnit.expect( count );
for ( lang in allTests ) {
tests = allTests[ lang ];
for ( i = 0, len = allTests[ lang ].length; i < len; i++ ) {
test = tests[ i ];
+ if ( test.skip ) {
+ continue;
+ }
testData = fs.readFileSync( __dirname + '/data/' +
test.source, 'utf8' );
+
segmenter = new CX.Segmenter( testData, lang );
segmenter.segment();
- result = segmenter.getSegmentedContent();
- result = result.replace( /(\r\n|\n|\t|\r)/gm, '' );
- expectedResultData = fs.readFileSync( __dirname +
'/data/' + test.result, 'utf8' );
- expectedResultData = expectedResultData.replace(
/(\r\n|\n|\t|\r)/gm, '' );
- assert.strictEqual( result, expectedResultData,
test.desc || '' );
+ result = normalize( segmenter.getSegmentedContent() );
+ expectedResultData = normalize(
+ fs.readFileSync( __dirname + '/data/' +
test.result, 'utf8' )
+ );
+ assert.strictEqual( result, expectedResultData,
test.source + ': ' + test.desc || '' );
}
}
+ if ( skipCount > 0 ) {
+ console.warn( 'Skipped ' + skipCount + ' tests' );
+ }
} );
diff --git a/tests/segmentation/SegmentationTests.json
b/tests/segmentation/SegmentationTests.json
index aec60ca..527a1b5 100644
--- a/tests/segmentation/SegmentationTests.json
+++ b/tests/segmentation/SegmentationTests.json
@@ -13,7 +13,8 @@
{
"desc": "Exclamation, punctuation test",
"source": "test-2.html",
- "result": "result-2.html"
+ "result": "result-2.html",
+ "skip": true
},
{
"desc": "Multiple sentences",
@@ -66,7 +67,7 @@
"result": "result-12.html"
},
{
- "desc": "References can appear after period and space.
Example: Hydrogen is a gas. [1] It is .... In this case we dont have any choice
than considering [1] as part of second sentence",
+ "desc": "References can appear after period and space.
Example: Hydrogen is a gas. [1] It is ...",
"source": "test-13.html",
"result": "result-13.html"
},
@@ -81,9 +82,19 @@
"result": "result-15.html"
},
{
+ "desc": "Three sentences in a single text node",
+ "source": "test-16.html",
+ "result": "result-16.html"
+ },
+ {
"desc": "Paragraph ending with reference and already
having reference in between. The second reference should not be identified as
repeating reference. If identified as repeating reference, 7 letters from last
word will be missing.",
"source":
"test-ends-with-references-missing-letters.html",
"result":
"result-ends-with-references-missing-letters.html"
+ },
+ {
+ "desc": "Sentences ending with abbreviations",
+ "source": "test-17.html",
+ "result": "result-17.html"
}
],
"hi": [
diff --git a/tests/segmentation/data/result-10.html
b/tests/segmentation/data/result-10.html
index 313e71c..39d3ef6 100644
--- a/tests/segmentation/data/result-10.html
+++ b/tests/segmentation/data/result-10.html
@@ -1,6 +1,6 @@
<p id="0">
<span class="cx-segment" data-segmentid="1">Sentence one
<span class="reference">
- <a class="cx-link" data-linkid="2" href="#">1</a>
+ <a href="#">1</a>
</span>and rest of sentence</span>
</p>
diff --git a/tests/segmentation/data/result-11.html
b/tests/segmentation/data/result-11.html
index 969c3c1..ebb8480 100644
--- a/tests/segmentation/data/result-11.html
+++ b/tests/segmentation/data/result-11.html
@@ -1,12 +1,12 @@
<p id="0">
<span class="cx-segment" data-segmentid="1">
Hydrogen's atomic number is one.<span class="reference">
- <a class="cx-link" data-linkid="2" href="#">1</a>
+ <a href="#">1</a>
</span>
</span>
- <span class="cx-segment" data-segmentid="3">Hydrogen is a gas
+ <span class="cx-segment" data-segmentid="2">Hydrogen is a gas
<span class="reference">
- <a class="cx-link" data-linkid="4" href="#">2</a>
+ <a href="#">2</a>
</span>and it is
</span>
</p>
diff --git a/tests/segmentation/data/result-12.html
b/tests/segmentation/data/result-12.html
index 7c0156a..25c6e70 100644
--- a/tests/segmentation/data/result-12.html
+++ b/tests/segmentation/data/result-12.html
@@ -1,11 +1,11 @@
<p id="0">
<span class="cx-segment" data-segmentid="1">Sentence one <span
class="reference">
- <a class="cx-link" data-linkid="2" href="#">1</a>
+ <a href="#">1</a>
</span>
<span class="reference">
- <a class="cx-link" data-linkid="3" href="#">2</a>
+ <a href="#">2</a>
</span>
<span class="reference">
- <a class="cx-link" data-linkid="4" href="#">3</a>
+ <a href="#">3</a>
</span> and rest of sentence</span>
</p>
diff --git a/tests/segmentation/data/result-13.html
b/tests/segmentation/data/result-13.html
index 26ec46d..fafff5f 100644
--- a/tests/segmentation/data/result-13.html
+++ b/tests/segmentation/data/result-13.html
@@ -1,7 +1,8 @@
<p id="0">
- <span class="cx-segment" data-segmentid="1">Sentence one. </span>
- <span class="cx-segment" data-segmentid="2">
- <span class="reference">
- <a class="cx-link" data-linkid="3"
href="#">reference</a>
- </span> Starts with reference</span>
+ <span class="cx-segment" data-segmentid="1">Sentence one.
+ <span class="reference">
+ <a href="#">reference</a>
+ </span>
+ </span>
+ <span class="cx-segment" data-segmentid="2">Starts with reference</span>
</p>
diff --git a/tests/segmentation/data/result-15.html
b/tests/segmentation/data/result-15.html
index b36bb5b..42ab78a 100644
--- a/tests/segmentation/data/result-15.html
+++ b/tests/segmentation/data/result-15.html
@@ -1 +1 @@
-<p id="0"><span class="cx-segment" data-segmentid="1">When the GNU project
first started they "had an <a class="cx-link" data-linkid="2"
href="/wiki/Emacs" title="Emacs">Emacs</a> text editor with <a class="cx-link"
data-linkid="3" href="/wiki/Lisp_(programming_language)" title="Lisp
(programming language)">Lisp</a> for writing editor commands, a source level <a
class="cx-link" data-linkid="4" href="/wiki/Debugger"
title="Debugger">debugger</a>, a <a class="cx-link" data-linkid="5"
href="/wiki/Yacc" title="Yacc" data-original-title="">yacc</a>-compatible <a
class="cx-link" data-linkid="6" href="/wiki/Parsing" title="Parsing">parser</a>
generator, and a <a class="cx-link" data-linkid="7"
href="/wiki/Linker_(computing)" title="Linker (computing)">linker</a>".<span
id="cite_ref-4" class="reference"><a class="cx-link" data-linkid="8"
href="#cite_note-4">[4]</a></span></span><span class="cx-segment"
data-segmentid="9"> The GNU system required its own C compiler and tools to be
free software, so that these also had to be developed. </span><span
class="cx-segment" data-segmentid="10">By June 1987 the project had accumulated
and developed free software for an assembler, an almost finished portable
optimizing C compiler (<a class="cx-link" data-linkid="11"
href="/wiki/GNU_Compiler_Collection" title="GNU Compiler Collection"
data-original-title="">GCC</a>), an editor (<a class="cx-link" data-linkid="12"
href="/wiki/Emacs" title="Emacs">GNU Emacs</a>), and various Unix utilities
(such as <code>ls</code>, <code>grep</code>, <code>awk</code>,
<code>make</code> and <code>ld</code>).<span id="cite_ref-5"
class="reference"><a class="cx-link" data-linkid="13" href="#cite_note-5"
title="" data-original-title="">[5]</a></span></span><span class="cx-segment"
data-segmentid="14"> They had an initial kernel that needed more
updates.</span></p>
+<p id="0"><span class="cx-segment" data-segmentid="1">When the GNU project
first started they "had an <a class="cx-link" data-linkid="2"
href="/wiki/Emacs" title="Emacs">Emacs</a> text editor with <a class="cx-link"
data-linkid="3" href="/wiki/Lisp_(programming_language)" title="Lisp
(programming language)">Lisp</a> for writing editor commands, a source level <a
class="cx-link" data-linkid="4" href="/wiki/Debugger"
title="Debugger">debugger</a>, a <a class="cx-link" data-linkid="5"
href="/wiki/Yacc" title="Yacc" data-original-title="">yacc</a>-compatible <a
class="cx-link" data-linkid="6" href="/wiki/Parsing" title="Parsing">parser</a>
generator, and a <a class="cx-link" data-linkid="7"
href="/wiki/Linker_(computing)" title="Linker (computing)">linker</a>".<span
id="cite_ref-4" class="reference"><a href="#cite_note-4">[4]</a></span>
</span><span class="cx-segment" data-segmentid="8">The GNU system required its
own C compiler and tools to be free software, so that these also had to be
developed. </span><span class="cx-segment" data-segmentid="9">By June 1987 the
project had accumulated and developed free software for an assembler, an almost
finished portable optimizing C compiler (<a class="cx-link" data-linkid="10"
href="/wiki/GNU_Compiler_Collection" title="GNU Compiler Collection"
data-original-title="">GCC</a>), an editor (<a class="cx-link" data-linkid="11"
href="/wiki/Emacs" title="Emacs">GNU Emacs</a>), and various Unix utilities
(such as <code>ls</code>, <code>grep</code>, <code>awk</code>,
<code>make</code> and <code>ld</code>).<span id="cite_ref-5"
class="reference"><a href="#cite_note-5" title=""
data-original-title="">[5]</a></span> </span><span class="cx-segment"
data-segmentid="12">They had an initial kernel that needed more
updates.</span></p>
diff --git a/tests/segmentation/data/result-16.html
b/tests/segmentation/data/result-16.html
new file mode 100644
index 0000000..4e80057
--- /dev/null
+++ b/tests/segmentation/data/result-16.html
@@ -0,0 +1 @@
+<p id="0"><span class="cx-segment" data-segmentid="1">Yes. </span><span
class="cx-segment" data-segmentid="2">No. </span><span class="cx-segment"
data-segmentid="3">Maybe.</span></p>
diff --git a/tests/segmentation/data/result-17.html
b/tests/segmentation/data/result-17.html
new file mode 100644
index 0000000..7906c02
--- /dev/null
+++ b/tests/segmentation/data/result-17.html
@@ -0,0 +1 @@
+<p id="0"><span class="cx-segment" data-segmentid="1">Some in the UK.
</span><span class="cx-segment" data-segmentid="2">Others in the US.</span></p>
diff --git a/tests/segmentation/data/result-7.html
b/tests/segmentation/data/result-7.html
index 9e104ec..f4a5e67 100644
--- a/tests/segmentation/data/result-7.html
+++ b/tests/segmentation/data/result-7.html
@@ -1,7 +1,7 @@
<figure id="0">
<span class="cx-segment" data-segmentid="1">
<a class="cx-link" data-linkid="2" href="#">
- <img src="img.png"></img>
+ <img src="img.png"/>
</a>
</span>
<figcaption id="3">
diff --git a/tests/segmentation/data/result-8.html
b/tests/segmentation/data/result-8.html
index 6996f4d..301387c 100644
--- a/tests/segmentation/data/result-8.html
+++ b/tests/segmentation/data/result-8.html
@@ -1,8 +1,8 @@
<p id="0">
<span class="cx-segment" data-segmentid="1">Sentence one.
<span class="reference">
- <a class="cx-link" data-linkid="2"
href="#">reference</a>
+ <a href="#">reference</a>
</span>
- </span>
- <span class="cx-segment" data-segmentid="3"> Starts with
reference</span>
+ </span>
+ <span class="cx-segment" data-segmentid="2">Starts with reference</span>
</p>
diff --git a/tests/segmentation/data/result-9.html
b/tests/segmentation/data/result-9.html
index 91f99b4..e86493c 100644
--- a/tests/segmentation/data/result-9.html
+++ b/tests/segmentation/data/result-9.html
@@ -1,14 +1,14 @@
<p id="0">
<span class="cx-segment" data-segmentid="1">Sentence one.
<span class="reference">
- <a class="cx-link" data-linkid="2" href="#">1</a>
+ <a href="#">1</a>
</span>
<span class="reference">
- <a class="cx-link" data-linkid="3" href="#">2</a>
+ <a href="#">2</a>
</span>
<span class="reference">
- <a class="cx-link" data-linkid="4" href="#">3</a>
+ <a href="#">3</a>
</span>
- </span>
- <span class="cx-segment" data-segmentid="5"> Starts with
reference</span>
+ </span>
+ <span class="cx-segment" data-segmentid="2">Starts with reference</span>
</p>
diff --git a/tests/segmentation/data/result-debian-1.html
b/tests/segmentation/data/result-debian-1.html
index afdde28..b8cb1c8 100644
--- a/tests/segmentation/data/result-debian-1.html
+++ b/tests/segmentation/data/result-debian-1.html
@@ -7,18 +7,18 @@
<a class="cx-link" data-linkid="4" rel="mw:WikiLink"
href="./Xfce">Xfce</a>and
<a class="cx-link" data-linkid="5" rel="mw:WikiLink"
href="./LXDE">LXDE</a>.
<span about="#mwt72" class="reference">
- <a class="cx-link" data-linkid="6"
href="#cite_note-25">[25]</a>
+ <a href="#cite_note-25">[25]</a>
</span>
<span about="#mwt72" class="reference">
- <a class="cx-link" data-linkid="7"
href="#cite_note-25">[26]</a>
+ <a href="#cite_note-25">[26]</a>
</span>
- </span>
- <span class="cx-segment" data-segmentid="8"> Less common
- <a class="cx-link" data-linkid="9" rel="mw:WikiLink"
href="./Window_manager">window managers</a>such as
- <a class="cx-link" data-linkid="10" rel="mw:WikiLink"
href="./Enlightenment_(window_manager)">Enlightenment</a>,
- <a class="cx-link" data-linkid="11" rel="mw:WikiLink"
href="./Openbox">Openbox</a>,
- <a class="cx-link" data-linkid="12" rel="mw:WikiLink"
href="./Fluxbox">Fluxbox</a>,
- <a class="cx-link" data-linkid="13" rel="mw:WikiLink"
href="./GNUstep">GNUstep</a>,
- <a class="cx-link" data-linkid="14" rel="mw:WikiLink"
href="./IceWM">IceWM</a>,
- <a class="cx-link" data-linkid="15" rel="mw:WikiLink"
href="./Window_Maker">Window Maker</a>and others can also be installed.</span>
+ </span>
+ <span class="cx-segment" data-segmentid="6">Less common
+ <a class="cx-link" data-linkid="7" rel="mw:WikiLink"
href="./Window_manager">window managers</a>such as
+ <a class="cx-link" data-linkid="8" rel="mw:WikiLink"
href="./Enlightenment_(window_manager)">Enlightenment</a>,
+ <a class="cx-link" data-linkid="9" rel="mw:WikiLink"
href="./Openbox">Openbox</a>,
+ <a class="cx-link" data-linkid="10" rel="mw:WikiLink"
href="./Fluxbox">Fluxbox</a>,
+ <a class="cx-link" data-linkid="11" rel="mw:WikiLink"
href="./GNUstep">GNUstep</a>,
+ <a class="cx-link" data-linkid="12" rel="mw:WikiLink"
href="./IceWM">IceWM</a>,
+ <a class="cx-link" data-linkid="13" rel="mw:WikiLink"
href="./Window_Maker">Window Maker</a>and others can also be installed.</span>
</p>
diff --git
a/tests/segmentation/data/result-ends-with-references-missing-letters.html
b/tests/segmentation/data/result-ends-with-references-missing-letters.html
index 4af0e04..a7e516f 100644
--- a/tests/segmentation/data/result-ends-with-references-missing-letters.html
+++ b/tests/segmentation/data/result-ends-with-references-missing-letters.html
@@ -1 +1 @@
-<p id="0"><span class="cx-segment" data-segmentid="1">By June 1987 the project
had accumulated and developed free software for an assembler, an almost
finished portable optimizing C compiler (<a class="cx-link" data-linkid="2"
href="/wiki/GNU_Compiler_Collection" title="GNU Compiler Collection"
data-original-title="">GCC</a>), an editor (<a class="cx-link" data-linkid="3"
href="/wiki/Emacs" title="Emacs">GNU Emacs</a>), and various Unix utilities
(such as ls, grep, awk, make and ld).<span id="cite_ref-5" class="reference"><a
class="cx-link" data-linkid="4" href="#cite_note-5" title=""
data-original-title="">[5]</a></span></span><span class="cx-segment"
data-segmentid="5"> They had an initial kernel that needed more updates.
</span><span class="cx-segment" data-segmentid="6">By June 1987 the project had
accumulated and developed free software for an assembler, an almost finished
portable optimizing C compiler (<a class="cx-link" data-linkid="7"
href="/wiki/GNU_Compiler_Collection" title="GNU Compiler Collection"
data-original-title="">GCC</a>), an editor (<a class="cx-link" data-linkid="8"
href="/wiki/Emacs" title="Emacs">GNU Emacs</a>), and various Unix utilities
(such as ls, grep, awk, make and ld.<span id="cite_ref-5" class="reference"><a
class="cx-link" data-linkid="9" href="#cite_note-5" title=""
data-original-title="">[6]</a></span></span></p>
+<p id="0"><span class="cx-segment" data-segmentid="1">By June 1987 the project
had accumulated and developed free software for an assembler, an almost
finished portable optimizing C compiler (<a class="cx-link" data-linkid="2"
href="/wiki/GNU_Compiler_Collection" title="GNU Compiler Collection"
data-original-title="">GCC</a>), an editor (<a class="cx-link" data-linkid="3"
href="/wiki/Emacs" title="Emacs">GNU Emacs</a>), and various Unix utilities
(such as ls, grep, awk, make and ld).<span id="cite_ref-5" class="reference"><a
href="#cite_note-5" title="" data-original-title="">[5]</a></span> </span><span
class="cx-segment" data-segmentid="4">They had an initial kernel that needed
more updates. </span><span class="cx-segment" data-segmentid="5">By June 1987
the project had accumulated and developed free software for an assembler, an
almost finished portable optimizing C compiler (<a class="cx-link"
data-linkid="6" href="/wiki/GNU_Compiler_Collection" title="GNU Compiler
Collection" data-original-title="">GCC</a>), an editor (<a class="cx-link"
data-linkid="7" href="/wiki/Emacs" title="Emacs">GNU Emacs</a>), and various
Unix utilities (such as ls, grep, awk, make and ld.<span id="cite_ref-5"
class="reference"><a href="#cite_note-5" title=""
data-original-title="">[6]</a></span></span></p>
diff --git a/tests/segmentation/data/test-16.html
b/tests/segmentation/data/test-16.html
new file mode 100644
index 0000000..9fb51f0
--- /dev/null
+++ b/tests/segmentation/data/test-16.html
@@ -0,0 +1 @@
+<p>Yes. No. Maybe.</p>
diff --git a/tests/segmentation/data/test-17.html
b/tests/segmentation/data/test-17.html
new file mode 100644
index 0000000..26d3af2
--- /dev/null
+++ b/tests/segmentation/data/test-17.html
@@ -0,0 +1 @@
+<p>Some in the UK. Others in the US.</p>
diff --git a/tests/segmentation/data/test-7.html
b/tests/segmentation/data/test-7.html
index fda9595..f6f3340 100644
--- a/tests/segmentation/data/test-7.html
+++ b/tests/segmentation/data/test-7.html
@@ -1,6 +1,6 @@
<figure>
<a href="#">
- <img src="img.png">
+ <img src="img.png"/>
</a>
<figcaption>Figure caption</figcaption>
</figure>
--
To view, visit https://gerrit.wikimedia.org/r/138532
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Iee6bfa499c9f23269a290df5de5e027e48746971
Gerrit-PatchSet: 8
Gerrit-Project: mediawiki/services/cxserver
Gerrit-Branch: master
Gerrit-Owner: Divec <[email protected]>
Gerrit-Reviewer: Divec <[email protected]>
Gerrit-Reviewer: Santhosh <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
