Marcoil has uploaded a new change for review.
https://gerrit.wikimedia.org/r/97733
Change subject: Bug 56490: Use the HTTP API in round-trip testing
......................................................................
Bug 56490: Use the HTTP API in round-trip testing
roundtrip-test.js now uses a (configurable) Parsoid HTTP API to obtain html
and wikitext for a page before comparing to the original wikitext.
The round-trip test client also has a new configuration option for the URL of
the Parsoid API.
Change-Id: I2cf651ede1762ae5bbf53127c7ecdff75003c996
---
M js/tests/client/client.js
M js/tests/client/config.example.js
M js/tests/roundtrip-test.js
3 files changed, 101 insertions(+), 46 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Parsoid
refs/changes/33/97733/1
diff --git a/js/tests/client/client.js b/js/tests/client/client.js
index ba9f96f..084c38c 100755
--- a/js/tests/client/client.js
+++ b/js/tests/client/client.js
@@ -78,7 +78,8 @@
rtTest.fetch( title, callback, {
setup: config.setup,
prefix: prefix,
- editMode: false
+ editMode: false,
+ parsoidURL: config.parsoidURL
} );
} catch ( err ) {
// Log it to console (for gabriel to watch scroll by)
diff --git a/js/tests/client/config.example.js
b/js/tests/client/config.example.js
index 923ae8f..0b6505d 100644
--- a/js/tests/client/config.example.js
+++ b/js/tests/client/config.example.js
@@ -30,7 +30,10 @@
// Insert the interwiki prefix for a localhost wiki
parsoidConfig.setInterwiki( 'localhost',
'http://localhost/wiki/api.php' );
- }
+ },
+
+ // The parsoid API to use
+ parsoidURL: 'http://parsoid.wmflabs.org/'
};
}
diff --git a/js/tests/roundtrip-test.js b/js/tests/roundtrip-test.js
index 2c0859a..9d1622c 100755
--- a/js/tests/roundtrip-test.js
+++ b/js/tests/roundtrip-test.js
@@ -2,13 +2,15 @@
"use strict";
var jsDiff = require( 'diff' ),
+ http = require( 'http' ),
optimist = require( 'optimist' ),
+ querystring = require( 'querystring' ),
domino = require( 'domino' ),
+ url = require( 'url' ),
zlib = require( 'zlib' ),
Util = require( '../lib/mediawiki.Util.js' ).Util,
DU = require( '../lib/mediawiki.DOMUtils.js' ).DOMUtils,
- WikitextSerializer = require(
'../lib/mediawiki.WikitextSerializer.js').WikitextSerializer,
TemplateRequest = require( '../lib/mediawiki.ApiRequest.js'
).TemplateRequest,
ParsoidConfig = require( '../lib/mediawiki.ParsoidConfig'
).ParsoidConfig,
MWParserEnvironment = require( '../lib/mediawiki.parser.environment.js'
).MWParserEnvironment;
@@ -380,8 +382,6 @@
};
var doubleRoundtripDiff = function ( env, offsets, body, out, cb ) {
- var src = env.page.src;
-
if ( offsets.length > 0 ) {
env.setPageSrcInfo( out );
env.errCB = function ( error ) {
@@ -390,7 +390,7 @@
};
var parserPipeline = Util.getParserPipeline( env,
'text/x-mediawiki/full' );
- parserPipeline.on( 'document', checkIfSignificant.bind( null,
env, offsets, src, body, out, cb ) );
+ parserPipeline.on( 'document', checkIfSignificant.bind( null,
env, offsets, env.page.src, body, out, cb ) );
parserPipeline.processToplevelDoc( out );
} else {
@@ -398,28 +398,71 @@
}
};
-var roundTripDiff = function ( env, document, cb ) {
- var out, diff, offsetPairs;
+var parsoidPost = function ( env, parsoidURL, prefix, title, text, oldid, cb )
{
+ var options = url.parse( url.resolve( parsoidURL, prefix + '/' + title
) );
+ options.method = 'POST';
+ options.headers = {
+ 'Content-Type': 'application/x-www-form-urlencoded',
+ };
- // Re-parse the HTML to uncover foster-parenting issues
- var origBody = document.body;
- document = domino.createDocument(DU.serializeNode(document));
+ var req = http.request( options, function( res ) {
+ res.setEncoding( 'utf8' );
+ res.body = '';
+
+ res.on( 'data', function ( chunk ) { res.body += chunk; } );
+
+ res.on( 'end', function () {
+ if ( res.statusCode !== 200 ) {
+ cb( res.body, null );
+ } else {
+ if ( env.profile ) {
+ // Record the time it's taken to parse
+ var timePrefix = oldid ? 'html2wt' :
'wt2html';
+ if ( res.headers[
'x-parsoid-performance' ] ) {
+ env.profile.time[ timePrefix ] =
+ parseInt( res.headers[
'x-parsoid-performance' ].
+ match(
/duration=((\d)+);/ )[1], 10 );
+ }
+ // Record the sizes
+ var sizePrefix = oldid ? 'wt' : 'html';
+ env.profile.size[ sizePrefix + 'raw' ] =
+ res.body.length;
+ // Compress to record the gzipped size
+ zlib.gzip( res.body, function( err,
gzippedbuf ) {
+ if ( !err ) {
+ env.profile.size[
sizePrefix + 'gzip' ] =
+
gzippedbuf.length;
+ }
+ cb( null, res );
+ } );
+ } else {
+ cb( null, res );
+ }
+ }
+ } );
+ } );
+
+ var data = {};
+ if ( oldid ) {
+ data.oldid = oldid;
+ data.html = text;
+ } else {
+ data.wt = text;
+ }
+ req.write( querystring.stringify( data ) );
+ req.end();
+};
+
+var roundTripDiff = function ( env, html, out, cb ) {
+ var diff, offsetPairs;
+
try {
- env.profile.time.serialize = new Date();
- out = new WikitextSerializer( { env: env }
).serializeDOM(document.body);
- env.profile.time.serialize = new Date() -
env.profile.time.serialize;
- env.profile.size.wikitext = out.length;
-
- // Finish the total time now
- if ( env.profile && env.profile.time ) {
- env.profile.time.total += new Date() -
env.profile.time.total_timer;
- delete( env.profile.time.total_timer );
- }
diff = jsDiff.diffLines( out, env.page.src );
offsetPairs = Util.convertDiffToOffsetPairs( diff );
if ( diff.length > 0 ) {
- doubleRoundtripDiff( env, offsetPairs, origBody, out,
cb );
+ var body = domino.createDocument( html ).body;
+ doubleRoundtripDiff( env, offsetPairs, body, out, cb );
} else {
cb( null, env, [] );
}
@@ -430,6 +473,11 @@
var fetch = function ( page, cb, options ) {
cb = typeof cb === 'function' ? cb : function () {};
+ var prefix = options.prefix || 'enwiki';
+
+ if ( options.apiURL ) {
+ prefix = 'customwiki';
+ }
var envCb = function ( err, env ) {
env.errCB = function ( error ) {
@@ -450,33 +498,31 @@
cb( err, env, [] );
} else {
env.setPageSrcInfo( src_and_metadata );
- env.profile.time.parse = new Date();
- Util.parse( env, function ( src, err, doc ) {
- env.profile.time.parse = new Date() -
env.profile.time.parse;
- if ( err ) {
- cb( err, env, [] );
- } else {
- // Pause the total time while
we compute these sizes
- env.profile.time.total += new
Date() - env.profile.time.total_timer;
- env.profile.size.htmlraw =
doc.outerHTML.length;
- zlib.gzip( doc.outerHTML,
function( err, buf ) {
- if ( !err ) {
-
env.profile.size.htmlgzip = buf.length;
- }
-
env.profile.time.total_timer = new Date();
- roundTripDiff( env,
doc, cb );
- });
- }
- }, err, env.page.src );
+ // First, fetch the HTML for the requested
page's wikitext
+ parsoidPost( env, options.parsoidURL, prefix,
page,
+ env.page.src, null, function ( err,
htmlRes ) {
+ if ( err ) {
+ cb( err, env, [] );
+ } else {
+ // And now, request the
wikitext for the obtained HTML
+ parsoidPost( env,
options.parsoidURL, prefix, page,
+ htmlRes.body,
src_and_metadata.revision.revid, function ( err, wtRes ) {
+ if (
err ) {
+
cb( err, env, [] );
+ } else {
+
// Finish the total time now
+
if ( env.profile && env.profile.time ) {
+
env.profile.time.total += new Date() - env.profile.time.total_timer;
+
delete( env.profile.time.total_timer );
+
}
+
roundTripDiff( env, htmlRes.body, wtRes.body, cb );
+ }
+ } );
+ }
+ } );
}
} );
};
-
- var prefix = options.prefix || 'enwiki';
-
- if ( options.apiURL ) {
- prefix = 'customwiki';
- }
var parsoidConfig = new ParsoidConfig( options, { defaultWiki: prefix }
);
@@ -552,6 +598,11 @@
description: 'Dump state (see below for supported dump
flags)',
'boolean': false,
'default': ""
+ },
+ 'parsoidURL': {
+ description: 'The URL for the Parsoid API',
+ 'boolean': false,
+ 'default': 'http://parsoid.wmflabs.org/'
}
});
--
To view, visit https://gerrit.wikimedia.org/r/97733
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I2cf651ede1762ae5bbf53127c7ecdff75003c996
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Parsoid
Gerrit-Branch: master
Gerrit-Owner: Marcoil <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
