Divec has uploaded a new change for review.
https://gerrit.wikimedia.org/r/138532
Change subject: Use LinearDoc for segmentation
......................................................................
Use LinearDoc for segmentation
Use LinearDoc to allow apply segmentation at a structural level.
Ensure valid, well-formed HTML output.
Include inter-segment whitespace with the previous segment.
Treat references as zero-width annotations; do not segment inside them.
Do not give link IDs to links inside a reference.
This revision makes no change to linear ID allocation (future task).
LinearDoc.js
* Add segment IDs to HTML output
* Add link IDs to HTML output
CXSegmenter.js
* Rewrite completely, using LinearDoc.js
SegmenterEn.js
SegmenterHi.js
* Basic plaintext segmenters that work well with LinearDoc segmentation
tests/segmentation/data/result*.html
* Rewrite for LinearDoc segmentation (remove some hardcoded limitations)
Change-Id: Iee6bfa499c9f23269a290df5de5e027e48746971
---
M lineardoc/LinearDoc.js
M segmentation/CXSegmenter.js
A segmentation/SegmenterEn.js
A segmentation/SegmenterHi.js
M tests/segmentation/SegmentationTests.json
M tests/segmentation/data/result-1.html
M tests/segmentation/data/result-10.html
M tests/segmentation/data/result-11.html
M tests/segmentation/data/result-12.html
M tests/segmentation/data/result-13.html
M tests/segmentation/data/result-14.html
M tests/segmentation/data/result-15.html
M tests/segmentation/data/result-2.html
M tests/segmentation/data/result-3.html
M tests/segmentation/data/result-4.html
M tests/segmentation/data/result-5.html
M tests/segmentation/data/result-6.html
M tests/segmentation/data/result-7.html
M tests/segmentation/data/result-8.html
M tests/segmentation/data/result-9.html
M tests/segmentation/data/result-debian-1.html
M tests/segmentation/data/result-ends-with-bracket.html
M tests/segmentation/data/result-ends-with-references-missing-letters.html
M tests/segmentation/data/test-10.html
M tests/segmentation/data/test-11.html
M tests/segmentation/data/test-12.html
M tests/segmentation/data/test-13.html
M tests/segmentation/data/test-8.html
M tests/segmentation/data/test-9.html
29 files changed, 211 insertions(+), 112 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/cxserver
refs/changes/32/138532/1
diff --git a/lineardoc/LinearDoc.js b/lineardoc/LinearDoc.js
index 54aba8f..f0b1c82 100644
--- a/lineardoc/LinearDoc.js
+++ b/lineardoc/LinearDoc.js
@@ -311,30 +311,65 @@
}
/**
+ * Set link IDs in-place on text chunks
+ *
+ * @private
+ * @param {TextChunk[]} textChunks Consecutive text chunks
+ * @param {Function} getNextId function accepting 'link' and returning next ID
+ */
+function setLinkIdsInPlace( textChunks, getNextId ) {
+ var i, iLen, j, jLen, tags, tag, href;
+ for ( i = 0, iLen = textChunks.length; i < iLen; i++ ) {
+ tags = textChunks[ i ].tags;
+ for ( j = 0, jLen = tags.length; j < jLen; j++ ) {
+ tag = tags[ j ];
+ if (
+ tag.name === 'a' &&
+ tag.attributes.href !== undefined &&
+ tag.attributes[ 'data-linkid' ] === undefined
+ ) {
+ // Hack: copy href, then remove it, then re-add
it, so that
+ // attributes appear in alphabetical order (ugh)
+ href = tag.attributes.href;
+ delete tag.attributes.href;
+ tag.attributes.class = 'cx-link';
+ tag.attributes[ 'data-linkid' ] = '' +
getNextId( 'link' );
+ tag.attributes.href = href;
+ }
+ }
+ }
+}
+
+/**
* Segment the text block into sentences
* @method
* @param {Function} getBoundaries Function taking plaintext, returning offset
array
+ * @param {Function} getNextId Function taking 'segment'|'link', returning
next ID
* @return {TextBlock} Segmented version, with added span tags
*/
-TextBlock.prototype.segment = function ( getBoundaries ) {
+TextBlock.prototype.segment = function ( getBoundaries, getNextId ) {
var i, len, textChunk, boundary, relOffset,
allTextChunks = [],
currentTextChunks = [],
+ modifiedTextChunks,
charCount = 0,
boundaries = getBoundaries( this.getPlainText() ),
- bPtr = 0,
- segId = 1;
+ bPtr = 0;
function flushChunks() {
if ( currentTextChunks.length > 0 ) {
- allTextChunks.push.apply( allTextChunks, addCommonTag(
- currentTextChunks, {
+ modifiedTextChunks = addCommonTag(
+ currentTextChunks,
+ {
name: 'span',
attributes: {
- class: 'seg s' + segId++
+ 'class': 'cx-segment',
+ 'data-segmentid': '' +
getNextId( 'segment' )
}
}
- ) );
+ );
+ setLinkIdsInPlace( modifiedTextChunks, getNextId );
+ allTextChunks.push.apply( allTextChunks,
modifiedTextChunks );
currentTextChunks = [];
}
}
@@ -461,14 +496,28 @@
*/
Doc.prototype.segment = function ( getBoundaries ) {
var i, len, item, textBlock,
- newDoc = new Doc();
+ newDoc = new Doc(),
+ nextId = 1;
+
+ // TODO: return different counters depending on type
+ function getNextId( type ) {
+ if ( type === 'segment' || type === 'link' ) {
+ return nextId++;
+ } else {
+ throw new Error( 'Unknown ID type: ' + type );
+ }
+ }
+
for ( i = 0, len = this.items.length; i < len; i++ ) {
item = this.items[ i ];
if ( this.items[ i ].type !== 'textblock' ) {
newDoc.addItem( item.type, item.item );
} else {
textBlock = item.item;
- newDoc.addItem( 'textblock', textBlock.segment(
getBoundaries ) );
+ newDoc.addItem(
+ 'textblock',
+ textBlock.segment( getBoundaries, getNextId )
+ );
}
}
return newDoc;
diff --git a/segmentation/CXSegmenter.js b/segmentation/CXSegmenter.js
index 3ba9e5e..fa01018 100644
--- a/segmentation/CXSegmenter.js
+++ b/segmentation/CXSegmenter.js
@@ -9,52 +9,37 @@
'use strict';
-var CXParserFactory = require( __dirname + '/CXParserFactory.js'
).CXParserFactory,
- $ = require( 'jquery' );
+var LinearDoc = require( '../lineardoc/LinearDoc' ),
+ getBoundariesEn = require( './SegmenterEn' ).getBoundaries,
+ getBoundariesHi = require( './SegmenterHi' ).getBoundaries;
+
+function getBoundaryFunction( language ) {
+ if ( language === 'en' ) {
+ return getBoundariesEn;
+ } else if ( language === 'hi' ) {
+ return getBoundariesHi;
+ } else {
+ throw new Error( 'No boundary function for language:' +
language );
+ }
+}
function CXSegmenter( content, language ) {
+ this.parser = new LinearDoc.Parser();
+ this.parser.init();
+ this.getBoundaries = getBoundaryFunction( language );
this.content = content;
- this.segments = {};
- this.segmentedContent = null;
- this.links = {};
- this.parser = ( new CXParserFactory() ).getParser( language || 'en' );
+ this.originalDoc = null;
+ this.segmentedDoc = null;
}
CXSegmenter.prototype.segment = function () {
- this.parse();
- this.extractSegments();
-};
-
-CXSegmenter.prototype.parse = function () {
- this.parser.parse( this.content );
- this.links = this.parser.links;
- this.segmentedContent = this.parser.segmentedContent;
-};
-
-CXSegmenter.prototype.getLinks = function () {
- return this.links;
-};
-
-CXSegmenter.prototype.extractSegments = function () {
- var segmenter = this,
- $container = $( '<div>' ).html( this.segmentedContent );
-
- $container.find( '.cx-segment' ).each( function ( index, section ) {
- var $section = $( section ),
- segmentId = $section.data( 'segmentid' );
-
- segmenter.segments[segmentId] = {
- source: $section.html()
- };
- } );
-};
-
-CXSegmenter.prototype.getSegments = function () {
- return this.segments;
+ this.parser.write( this.content );
+ this.originalDoc = this.parser.builder.doc;
+ this.segmentedDoc = this.originalDoc.segment( this.getBoundaries );
};
CXSegmenter.prototype.getSegmentedContent = function () {
- return this.segmentedContent;
+ return this.segmentedDoc.getHtml();
};
module.exports.CXSegmenter = CXSegmenter;
diff --git a/segmentation/SegmenterEn.js b/segmentation/SegmenterEn.js
new file mode 100644
index 0000000..cacc4b5
--- /dev/null
+++ b/segmentation/SegmenterEn.js
@@ -0,0 +1,36 @@
+var findAll = require( '../lineardoc/LinearDoc' ).findAll;
+
+/**
+ * Test a possible English sentence boundary match
+ *
+ * @param {string} text The plaintext to segment
+ * @param {Object} match The possible boundary match (returned by regex.exec)
+ * @return {number|null} The boundary offset, or null if not a sentence
boundary
+ */
+function findBoundaryEn( text, match ) {
+ var tail = text.slice( match.index + 1, text.length );
+ // Trailing non-final punctuation: not a sentence boundary
+ if ( tail.match( /^[,;:]/ ) ) {
+ return null;
+ }
+ // Next word character is number or lower-case: not a sentence boundary
+ if ( tail.match( /^\W*[0-9a-z]/ ) ) {
+ return null;
+ }
+ // Include any closing punctuation and trailing space
+ return match.index + 1 + tail.match( /^['”"’]*\s*/ )[0].length;
+}
+
+/**
+ * Find English sentence boundaries
+ *
+ * @param {string} text The plaintext to segment
+ * @returns {number[]} Sentence boundary offsets
+ */
+function getBoundaries( text ) {
+ // Regex to find possible English sentence boundaries.
+ // Must not use a shared regex instance (re.lastIndex is used)
+ return findAll( text, /[.!?]/g, findBoundaryEn );
+}
+
+module.exports = { getBoundaries: getBoundaries };
diff --git a/segmentation/SegmenterHi.js b/segmentation/SegmenterHi.js
new file mode 100644
index 0000000..03941ec
--- /dev/null
+++ b/segmentation/SegmenterHi.js
@@ -0,0 +1,28 @@
+var findAll = require( '../lineardoc/LinearDoc' ).findAll;
+
+/**
+ * Test a possible Hindi sentence boundary match
+ *
+ * @param {string} text The plaintext to segment
+ * @param {Object} match The possible boundary match (returned by regex.exec)
+ * @return {number|null} The boundary offset, or null if not a sentence
boundary
+ */
+function findBoundaryHi( text, match ) {
+ var tail = text.slice( match.index + 1, text.length );
+ // Include any trailing space
+ return match.index + 1 + tail.match( /^\s*/ )[0].length;
+}
+
+/**
+ * Find Hindi sentence boundaries
+ *
+ * @param {string} text The plaintext to segment
+ * @returns {number[]} Sentence boundary offsets
+ */
+function getBoundaries( text ) {
+ // Regex to find possible Hindi sentence boundaries.
+ // Must not use a shared regex instance (re.lastIndex is used)
+ return findAll( text, /[।!?]/g, findBoundaryHi );
+}
+
+module.exports = { getBoundaries: getBoundaries };
diff --git a/tests/segmentation/SegmentationTests.json
b/tests/segmentation/SegmentationTests.json
index aec60ca..616e03b 100644
--- a/tests/segmentation/SegmentationTests.json
+++ b/tests/segmentation/SegmentationTests.json
@@ -66,7 +66,7 @@
"result": "result-12.html"
},
{
- "desc": "References can appear after period and space.
Example: Hydrogen is a gas. [1] It is .... In this case we dont have any choice
than considering [1] as part of second sentence",
+ "desc": "References can appear after period and space.
Example: Hydrogen is a gas. [1] It is ...",
"source": "test-13.html",
"result": "result-13.html"
},
diff --git a/tests/segmentation/data/result-1.html
b/tests/segmentation/data/result-1.html
index c309b26..946364f 100644
--- a/tests/segmentation/data/result-1.html
+++ b/tests/segmentation/data/result-1.html
@@ -1,3 +1,3 @@
-<p id="0">
+<p>
<span class="cx-segment" data-segmentid="1">A simple paragraph.</span>
</p>
diff --git a/tests/segmentation/data/result-10.html
b/tests/segmentation/data/result-10.html
index 313e71c..d05d5b2 100644
--- a/tests/segmentation/data/result-10.html
+++ b/tests/segmentation/data/result-10.html
@@ -1,6 +1,6 @@
-<p id="0">
+<p>
<span class="cx-segment" data-segmentid="1">Sentence one
- <span class="reference">
- <a class="cx-link" data-linkid="2" href="#">1</a>
+ <span typeof="mw:Extension/ref">
+ <a href="#">1</a>
</span>and rest of sentence</span>
</p>
diff --git a/tests/segmentation/data/result-11.html
b/tests/segmentation/data/result-11.html
index 969c3c1..879d59b 100644
--- a/tests/segmentation/data/result-11.html
+++ b/tests/segmentation/data/result-11.html
@@ -1,12 +1,12 @@
-<p id="0">
+<p>
<span class="cx-segment" data-segmentid="1">
- Hydrogen's atomic number is one.<span class="reference">
- <a class="cx-link" data-linkid="2" href="#">1</a>
+ Hydrogen's atomic number is one.<span typeof="mw:Extension/ref">
+ <a href="#">1</a>
</span>
</span>
- <span class="cx-segment" data-segmentid="3">Hydrogen is a gas
- <span class="reference">
- <a class="cx-link" data-linkid="4" href="#">2</a>
+ <span class="cx-segment" data-segmentid="2">Hydrogen is a gas
+ <span typeof="mw:Extension/ref">
+ <a href="#">2</a>
</span>and it is
</span>
</p>
diff --git a/tests/segmentation/data/result-12.html
b/tests/segmentation/data/result-12.html
index 7c0156a..db9bf2c 100644
--- a/tests/segmentation/data/result-12.html
+++ b/tests/segmentation/data/result-12.html
@@ -1,11 +1,11 @@
-<p id="0">
- <span class="cx-segment" data-segmentid="1">Sentence one <span
class="reference">
- <a class="cx-link" data-linkid="2" href="#">1</a>
+<p>
+ <span class="cx-segment" data-segmentid="1">Sentence one <span
typeof="mw:Extension/ref">
+ <a href="#">1</a>
</span>
- <span class="reference">
- <a class="cx-link" data-linkid="3" href="#">2</a>
+ <span typeof="mw:Extension/ref">
+ <a href="#">2</a>
</span>
- <span class="reference">
- <a class="cx-link" data-linkid="4" href="#">3</a>
+ <span typeof="mw:Extension/ref">
+ <a href="#">3</a>
</span> and rest of sentence</span>
</p>
diff --git a/tests/segmentation/data/result-13.html
b/tests/segmentation/data/result-13.html
index 26ec46d..5015e4a 100644
--- a/tests/segmentation/data/result-13.html
+++ b/tests/segmentation/data/result-13.html
@@ -1,7 +1,8 @@
-<p id="0">
- <span class="cx-segment" data-segmentid="1">Sentence one. </span>
- <span class="cx-segment" data-segmentid="2">
- <span class="reference">
- <a class="cx-link" data-linkid="3"
href="#">reference</a>
- </span> Starts with reference</span>
+<p>
+ <span class="cx-segment" data-segmentid="1">Sentence one.
+ <span typeof="mw:Extension/ref">
+ <a href="#">reference</a>
+ </span>
+ </span>
+ <span class="cx-segment" data-segmentid="2">Starts with reference</span>
</p>
diff --git a/tests/segmentation/data/result-14.html
b/tests/segmentation/data/result-14.html
index 6e4f0fe..610764a 100644
--- a/tests/segmentation/data/result-14.html
+++ b/tests/segmentation/data/result-14.html
@@ -1,4 +1,4 @@
-<p id="0">
+<p>
<span class="cx-segment" data-segmentid="1">माउज़र पिस्तौल (
<a class="cx-link" data-linkid="2" href="#">अंग्रेजी</a>:
Mauser C96) मूल रूप से जर्मनी में बनी एक अर्द्ध स्वचालित पिस्तौल है। </span>
<span class="cx-segment" data-segmentid="3">इस पिस्तौल का डिजाइन जर्मनी
निवासी दो माउज़र बन्धुओं ने सन् 1895 में तैयार किया था।</span>
diff --git a/tests/segmentation/data/result-15.html
b/tests/segmentation/data/result-15.html
index b36bb5b..d756071 100644
--- a/tests/segmentation/data/result-15.html
+++ b/tests/segmentation/data/result-15.html
@@ -1 +1 @@
-<p id="0"><span class="cx-segment" data-segmentid="1">When the GNU project
first started they "had an <a class="cx-link" data-linkid="2"
href="/wiki/Emacs" title="Emacs">Emacs</a> text editor with <a class="cx-link"
data-linkid="3" href="/wiki/Lisp_(programming_language)" title="Lisp
(programming language)">Lisp</a> for writing editor commands, a source level <a
class="cx-link" data-linkid="4" href="/wiki/Debugger"
title="Debugger">debugger</a>, a <a class="cx-link" data-linkid="5"
href="/wiki/Yacc" title="Yacc" data-original-title="">yacc</a>-compatible <a
class="cx-link" data-linkid="6" href="/wiki/Parsing" title="Parsing">parser</a>
generator, and a <a class="cx-link" data-linkid="7"
href="/wiki/Linker_(computing)" title="Linker (computing)">linker</a>".<span
id="cite_ref-4" class="reference"><a class="cx-link" data-linkid="8"
href="#cite_note-4">[4]</a></span></span><span class="cx-segment"
data-segmentid="9"> The GNU system required its own C compiler and tools to be
free software, so that these also had to be developed. </span><span
class="cx-segment" data-segmentid="10">By June 1987 the project had accumulated
and developed free software for an assembler, an almost finished portable
optimizing C compiler (<a class="cx-link" data-linkid="11"
href="/wiki/GNU_Compiler_Collection" title="GNU Compiler Collection"
data-original-title="">GCC</a>), an editor (<a class="cx-link" data-linkid="12"
href="/wiki/Emacs" title="Emacs">GNU Emacs</a>), and various Unix utilities
(such as <code>ls</code>, <code>grep</code>, <code>awk</code>,
<code>make</code> and <code>ld</code>).<span id="cite_ref-5"
class="reference"><a class="cx-link" data-linkid="13" href="#cite_note-5"
title="" data-original-title="">[5]</a></span></span><span class="cx-segment"
data-segmentid="14"> They had an initial kernel that needed more
updates.</span></p>
+<p><span class="cx-segment" data-segmentid="1">When the GNU project first
started they "had an <a class="cx-link" data-linkid="2" href="/wiki/Emacs"
title="Emacs">Emacs</a> text editor with <a class="cx-link" data-linkid="3"
href="/wiki/Lisp_(programming_language)" title="Lisp (programming
language)">Lisp</a> for writing editor commands, a source level <a
class="cx-link" data-linkid="4" href="/wiki/Debugger"
title="Debugger">debugger</a>, a <a class="cx-link" data-linkid="5"
href="/wiki/Yacc" title="Yacc" data-original-title="">yacc</a>-compatible <a
class="cx-link" data-linkid="6" href="/wiki/Parsing" title="Parsing">parser</a>
generator, and a <a class="cx-link" data-linkid="7"
href="/wiki/Linker_(computing)" title="Linker (computing)">linker</a>".<span
id="cite_ref-4" class="reference"><a class="cx-link" data-linkid="8"
href="#cite_note-4">[4]</a></span></span><span class="cx-segment"
data-segmentid="9"> The GNU system required its own C compiler and tools to be
free software, so that these also had to be developed. </span><span
class="cx-segment" data-segmentid="10">By June 1987 the project had accumulated
and developed free software for an assembler, an almost finished portable
optimizing C compiler (<a class="cx-link" data-linkid="11"
href="/wiki/GNU_Compiler_Collection" title="GNU Compiler Collection"
data-original-title="">GCC</a>), an editor (<a class="cx-link" data-linkid="12"
href="/wiki/Emacs" title="Emacs">GNU Emacs</a>), and various Unix utilities
(such as <code>ls</code>, <code>grep</code>, <code>awk</code>,
<code>make</code> and <code>ld</code>).<span id="cite_ref-5"
class="reference"><a class="cx-link" data-linkid="13" href="#cite_note-5"
title="" data-original-title="">[5]</a></span></span><span class="cx-segment"
data-segmentid="14"> They had an initial kernel that needed more
updates.</span></p>
diff --git a/tests/segmentation/data/result-2.html
b/tests/segmentation/data/result-2.html
index 7f3c39c..feee10a 100644
--- a/tests/segmentation/data/result-2.html
+++ b/tests/segmentation/data/result-2.html
@@ -1,4 +1,4 @@
-<p id="0">
+<p>
<span class="cx-segment" data-segmentid="1">Hello! </span>
<span class="cx-segment" data-segmentid="2">Mr. D. John, How are
you?</span>
</p>
diff --git a/tests/segmentation/data/result-3.html
b/tests/segmentation/data/result-3.html
index 466af68..e78c156 100644
--- a/tests/segmentation/data/result-3.html
+++ b/tests/segmentation/data/result-3.html
@@ -1,4 +1,4 @@
-<p id="0">
+<p>
<span class="cx-segment" data-segmentid="1">This is first sentence.
</span>
<span class="cx-segment" data-segmentid="2">This is second
sentence.</span>
</p>
diff --git a/tests/segmentation/data/result-4.html
b/tests/segmentation/data/result-4.html
index e50988d..d5ec804 100644
--- a/tests/segmentation/data/result-4.html
+++ b/tests/segmentation/data/result-4.html
@@ -1,7 +1,7 @@
-<div id="0">
+<div>
<span class="cx-segment" data-segmentid="1">Some div</span>
</div>
-<p id="2">
- <span class="cx-segment" data-segmentid="3">This is first sentence.
</span>
- <span class="cx-segment" data-segmentid="4">This is second
sentence</span>
+<p>
+ <span class="cx-segment" data-segmentid="2">This is first sentence.
</span>
+ <span class="cx-segment" data-segmentid="3">This is second
sentence</span>
</p>
diff --git a/tests/segmentation/data/result-5.html
b/tests/segmentation/data/result-5.html
index 6fb53dd..8984387 100644
--- a/tests/segmentation/data/result-5.html
+++ b/tests/segmentation/data/result-5.html
@@ -1,4 +1,4 @@
-<p id="0">
+<p>
<span class="cx-segment" data-segmentid="1">
<a class="cx-link" data-linkid="2" href="#">Hydrogen</a>is a
gas</span>
</p>
diff --git a/tests/segmentation/data/result-6.html
b/tests/segmentation/data/result-6.html
index e7b092b..b5266a5 100644
--- a/tests/segmentation/data/result-6.html
+++ b/tests/segmentation/data/result-6.html
@@ -1,4 +1,4 @@
-<p id="0">
+<p>
<span class="cx-segment" data-segmentid="1">
<a class="cx-link" data-linkid="2" href="#">Hydrogen</a>is a
<a class="cx-link" data-linkid="3" href="#">gas</a>
diff --git a/tests/segmentation/data/result-7.html
b/tests/segmentation/data/result-7.html
index 9e104ec..dfa6015 100644
--- a/tests/segmentation/data/result-7.html
+++ b/tests/segmentation/data/result-7.html
@@ -1,10 +1,10 @@
-<figure id="0">
+<figure>
<span class="cx-segment" data-segmentid="1">
<a class="cx-link" data-linkid="2" href="#">
- <img src="img.png"></img>
+ <img src="img.png">
</a>
</span>
- <figcaption id="3">
- <span class="cx-segment" data-segmentid="4">Figure
caption</span>
+ <figcaption>
+ <span class="cx-segment" data-segmentid="3">Figure
caption</span>
</figcaption>
</figure>
diff --git a/tests/segmentation/data/result-8.html
b/tests/segmentation/data/result-8.html
index 6996f4d..cf84f32 100644
--- a/tests/segmentation/data/result-8.html
+++ b/tests/segmentation/data/result-8.html
@@ -1,8 +1,8 @@
-<p id="0">
+<p>
<span class="cx-segment" data-segmentid="1">Sentence one.
- <span class="reference">
- <a class="cx-link" data-linkid="2"
href="#">reference</a>
+ <span typeof="mw:Extension/ref">
+ <a href="#">reference</a>
</span>
- </span>
- <span class="cx-segment" data-segmentid="3"> Starts with
reference</span>
+ </span>
+ <span class="cx-segment" data-segmentid="2">Starts with reference</span>
</p>
diff --git a/tests/segmentation/data/result-9.html
b/tests/segmentation/data/result-9.html
index 91f99b4..b0c5c74 100644
--- a/tests/segmentation/data/result-9.html
+++ b/tests/segmentation/data/result-9.html
@@ -1,14 +1,14 @@
-<p id="0">
+<p>
<span class="cx-segment" data-segmentid="1">Sentence one.
- <span class="reference">
- <a class="cx-link" data-linkid="2" href="#">1</a>
+ <span typeof="mw:Extension/ref">
+ <a href="#">1</a>
</span>
- <span class="reference">
- <a class="cx-link" data-linkid="3" href="#">2</a>
+ <span typeof="mw:Extension/ref">
+ <a href="#">2</a>
</span>
- <span class="reference">
- <a class="cx-link" data-linkid="4" href="#">3</a>
+ <span typeof="mw:Extension/ref">
+ <a href="#">3</a>
</span>
- </span>
- <span class="cx-segment" data-segmentid="5"> Starts with
reference</span>
+ </span>
+ <span class="cx-segment" data-segmentid="2">Starts with reference</span>
</p>
diff --git a/tests/segmentation/data/result-debian-1.html
b/tests/segmentation/data/result-debian-1.html
index afdde28..925353b 100644
--- a/tests/segmentation/data/result-debian-1.html
+++ b/tests/segmentation/data/result-debian-1.html
@@ -1,4 +1,4 @@
-<p id="0">
+<p>
<span class="cx-segment" data-segmentid="1">Debian offers
<i>stable</i>and
<i>testing</i>CD images specifically built for
diff --git a/tests/segmentation/data/result-ends-with-bracket.html
b/tests/segmentation/data/result-ends-with-bracket.html
index cb1b849..9d78085 100644
--- a/tests/segmentation/data/result-ends-with-bracket.html
+++ b/tests/segmentation/data/result-ends-with-bracket.html
@@ -1 +1 @@
-<p id="0"><span class="cx-segment" data-segmentid="1">By June 1987 the project
had accumulated and developed free software for an assembler, an almost
finished portable optimizing C compiler (GCC), an editor (GNU Emacs), and
various Unix utilities (such as <code>ls</code>, <code>grep</code>,
<code>awk</code>, <code>make</code> and <code>ld</code>). </span><span
class="cx-segment" data-segmentid="2">They had an initial kernel that needed
more updates.</span></p>
+<p><span class="cx-segment" data-segmentid="1">By June 1987 the project had
accumulated and developed free software for an assembler, an almost finished
portable optimizing C compiler (GCC), an editor (GNU Emacs), and various Unix
utilities (such as <code>ls</code>, <code>grep</code>, <code>awk</code>,
<code>make</code> and <code>ld</code>). </span><span class="cx-segment"
data-segmentid="2">They had an initial kernel that needed more
updates.</span></p>
diff --git
a/tests/segmentation/data/result-ends-with-references-missing-letters.html
b/tests/segmentation/data/result-ends-with-references-missing-letters.html
index 4af0e04..2e17e5a 100644
--- a/tests/segmentation/data/result-ends-with-references-missing-letters.html
+++ b/tests/segmentation/data/result-ends-with-references-missing-letters.html
@@ -1 +1 @@
-<p id="0"><span class="cx-segment" data-segmentid="1">By June 1987 the project
had accumulated and developed free software for an assembler, an almost
finished portable optimizing C compiler (<a class="cx-link" data-linkid="2"
href="/wiki/GNU_Compiler_Collection" title="GNU Compiler Collection"
data-original-title="">GCC</a>), an editor (<a class="cx-link" data-linkid="3"
href="/wiki/Emacs" title="Emacs">GNU Emacs</a>), and various Unix utilities
(such as ls, grep, awk, make and ld).<span id="cite_ref-5" class="reference"><a
class="cx-link" data-linkid="4" href="#cite_note-5" title=""
data-original-title="">[5]</a></span></span><span class="cx-segment"
data-segmentid="5"> They had an initial kernel that needed more updates.
</span><span class="cx-segment" data-segmentid="6">By June 1987 the project had
accumulated and developed free software for an assembler, an almost finished
portable optimizing C compiler (<a class="cx-link" data-linkid="7"
href="/wiki/GNU_Compiler_Collection" title="GNU Compiler Collection"
data-original-title="">GCC</a>), an editor (<a class="cx-link" data-linkid="8"
href="/wiki/Emacs" title="Emacs">GNU Emacs</a>), and various Unix utilities
(such as ls, grep, awk, make and ld.<span id="cite_ref-5" class="reference"><a
class="cx-link" data-linkid="9" href="#cite_note-5" title=""
data-original-title="">[6]</a></span></span></p>
+<p><span class="cx-segment" data-segmentid="1">By June 1987 the project had
accumulated and developed free software for an assembler, an almost finished
portable optimizing C compiler (<a class="cx-link" data-linkid="2"
href="/wiki/GNU_Compiler_Collection" title="GNU Compiler Collection"
data-original-title="">GCC</a>), an editor (<a class="cx-link" data-linkid="3"
href="/wiki/Emacs" title="Emacs">GNU Emacs</a>), and various Unix utilities
(such as ls, grep, awk, make and ld).<span id="cite_ref-5" class="reference"><a
class="cx-link" data-linkid="4" href="#cite_note-5" title=""
data-original-title="">[5]</a></span></span><span class="cx-segment"
data-segmentid="5"> They had an initial kernel that needed more updates.
</span><span class="cx-segment" data-segmentid="6">By June 1987 the project had
accumulated and developed free software for an assembler, an almost finished
portable optimizing C compiler (<a class="cx-link" data-linkid="7"
href="/wiki/GNU_Compiler_Collection" title="GNU Compiler Collection"
data-original-title="">GCC</a>), an editor (<a class="cx-link" data-linkid="8"
href="/wiki/Emacs" title="Emacs">GNU Emacs</a>), and various Unix utilities
(such as ls, grep, awk, make and ld.<span id="cite_ref-5" class="reference"><a
class="cx-link" data-linkid="9" href="#cite_note-5" title=""
data-original-title="">[6]</a></span></span></p>
diff --git a/tests/segmentation/data/test-10.html
b/tests/segmentation/data/test-10.html
index 77acd2f..1b95c91 100644
--- a/tests/segmentation/data/test-10.html
+++ b/tests/segmentation/data/test-10.html
@@ -1,4 +1,4 @@
<p>Sentence one
- <span class="reference">
+ <span typeof="mw:Extension/ref">
<a href="#">1</a>
</span>and rest of sentence</p>
diff --git a/tests/segmentation/data/test-11.html
b/tests/segmentation/data/test-11.html
index 082695e..5949f37 100644
--- a/tests/segmentation/data/test-11.html
+++ b/tests/segmentation/data/test-11.html
@@ -1,7 +1,7 @@
<p>
- Hydrogen's atomic number is one.<span class="reference">
+ Hydrogen's atomic number is one.<span typeof="mw:Extension/ref">
<a href="#">1</a>
- </span>Hydrogen is a gas<span class="reference">
+ </span>Hydrogen is a gas<span typeof="mw:Extension/ref">
<a href="#">2</a>
</span>
and it is
diff --git a/tests/segmentation/data/test-12.html
b/tests/segmentation/data/test-12.html
index 84b4019..8265095 100644
--- a/tests/segmentation/data/test-12.html
+++ b/tests/segmentation/data/test-12.html
@@ -1,10 +1,10 @@
-<p>Sentence one <span class="reference">
+<p>Sentence one <span typeof="mw:Extension/ref">
<a href="#">1</a>
</span>
- <span class="reference">
+ <span typeof="mw:Extension/ref">
<a href="#">2</a>
</span>
- <span class="reference">
+ <span typeof="mw:Extension/ref">
<a href="#">3</a>
</span> and rest of sentence
</p>
diff --git a/tests/segmentation/data/test-13.html
b/tests/segmentation/data/test-13.html
index 413b752..f6ece2b 100644
--- a/tests/segmentation/data/test-13.html
+++ b/tests/segmentation/data/test-13.html
@@ -1 +1 @@
-<p>Sentence one. <span class="reference"><a href="#">reference</a></span>
Starts with reference</p>
+<p>Sentence one. <span typeof="mw:Extension/ref"><a
href="#">reference</a></span> Starts with reference</p>
diff --git a/tests/segmentation/data/test-8.html
b/tests/segmentation/data/test-8.html
index 1e4ceaf..30a210a 100644
--- a/tests/segmentation/data/test-8.html
+++ b/tests/segmentation/data/test-8.html
@@ -1 +1 @@
-<p>Sentence one.<span class="reference"><a href="#">reference</a></span>
Starts with reference</p>
+<p>Sentence one.<span typeof="mw:Extension/ref"><a
href="#">reference</a></span> Starts with reference</p>
diff --git a/tests/segmentation/data/test-9.html
b/tests/segmentation/data/test-9.html
index 93cbcdb..dd29b74 100644
--- a/tests/segmentation/data/test-9.html
+++ b/tests/segmentation/data/test-9.html
@@ -1,9 +1,9 @@
-<p>Sentence one.<span class="reference">
+<p>Sentence one.<span typeof="mw:Extension/ref">
<a href="#">1</a>
</span>
- <span class="reference">
+ <span typeof="mw:Extension/ref">
<a href="#">2</a>
</span>
- <span class="reference">
+ <span typeof="mw:Extension/ref">
<a href="#">3</a>
</span> Starts with reference</p>
--
To view, visit https://gerrit.wikimedia.org/r/138532
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Iee6bfa499c9f23269a290df5de5e027e48746971
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/cxserver
Gerrit-Branch: master
Gerrit-Owner: Divec <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
