Marcoil has uploaded a new change for review.
https://gerrit.wikimedia.org/r/99348
Change subject: Bug 56590: Use the Parsoid HTTP API in round-trip testing
......................................................................
Bug 56590: Use the Parsoid HTTP API in round-trip testing
roundtrip-test.js now uses a (configurable) Parsoid HTTP API to obtain html
and wikitext for a page before comparing to the original wikitext.
The round-trip test client also has a new configuration option for the URL of
the Parsoid API.
If no Parsoid URL is passed, the new module apiServer.js starts one for us.
Change-Id: Ibb9daa72d51d91c6192ac0d695e6ecc4e70005aa
---
A js/tests/apiServer.js
M js/tests/client/client.js
M js/tests/client/config.example.js
M js/tests/roundtrip-test.js
4 files changed, 164 insertions(+), 46 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Parsoid
refs/changes/48/99348/1
diff --git a/js/tests/apiServer.js b/js/tests/apiServer.js
new file mode 100644
index 0000000..2424a72
--- /dev/null
+++ b/js/tests/apiServer.js
@@ -0,0 +1,49 @@
+#!/usr/bin/env node
+"use strict";
+/**
+ * Manages a Parsoid server for testing.
+ */
+
+var child_process = require( 'child_process' );
+
+var forkedServer;
+
+/**
+ * Starts a Parsoid server on passed port or a random port if none passed,
+ * @return The URL of the created server
+ */
+var startParsoidServer = function ( port ) {
+ if ( !port ) {
+ port = 9000 + Math.floor( Math.random() * 100 );
+ }
+
+ forkedServer = child_process.fork( __dirname + '/../api/server.js',
+ [ '-c', '1' ], { env: { VCAP_APP_PORT: port } } );
+
+ // If this process dies, kill our Parsoid server
+ var weDied = function () {
+ forkedServer.removeListener( 'exit', startParsoidServer );
+ forkedServer.kill();
+ };
+ process.on( 'exit', weDied );
+
+ // If it dies on its own, restart it
+ forkedServer.on( 'exit', function( ) {
+ process.removeListener( 'exit', weDied );
+ startParsoidServer( port );
+ } );
+
+ return 'http://localhost:' + port.toString() + '/';
+};
+
+var stopParsoidServer = function () {
+ if ( forkedServer ) {
+ forkedServer.removeListener( 'exit', startParsoidServer );
+ forkedServer.kill();
+ }
+};
+
+module.exports = {
+ startParsoidServer: startParsoidServer,
+ stopParsoidServer: stopParsoidServer
+};
diff --git a/js/tests/client/client.js b/js/tests/client/client.js
index ba9f96f..7a658e3 100755
--- a/js/tests/client/client.js
+++ b/js/tests/client/client.js
@@ -7,12 +7,14 @@
var http = require( 'http' ),
qs = require( 'querystring' ),
exec = require( 'child_process' ).exec,
+ apiServer = require( '../apiServer.js' ),
commit, ctime,
lastCommit, lastCommitTime, lastCommitCheck,
repoPath = __dirname,
config = require( process.argv[2] || './config.js' ),
+ parsoidURL = config.parsoidURL,
rtTest = require( '../roundtrip-test.js' );
var getTitle = function( cb ) {
@@ -78,7 +80,8 @@
rtTest.fetch( title, callback, {
setup: config.setup,
prefix: prefix,
- editMode: false
+ editMode: false,
+ parsoidURL: parsoidURL
} );
} catch ( err ) {
// Log it to console (for gabriel to watch scroll by)
@@ -192,6 +195,11 @@
}
if ( module && !module.parent ) {
+ if ( !config.parsoidURL ) {
+ // If no Parsoid server was passed, start our own
+ parsoidURL = apiServer.startParsoidServer();
+ }
+
getGitCommit( function ( commitHash, commitTime ) {
commit = commitHash;
ctime = commitTime;
diff --git a/js/tests/client/config.example.js
b/js/tests/client/config.example.js
index 923ae8f..4d13d47 100644
--- a/js/tests/client/config.example.js
+++ b/js/tests/client/config.example.js
@@ -30,7 +30,10 @@
// Insert the interwiki prefix for a localhost wiki
parsoidConfig.setInterwiki( 'localhost',
'http://localhost/wiki/api.php' );
- }
+ },
+
+ // The parsoid API to use. If null, create our own server
+ parsoidURL: null
};
}
diff --git a/js/tests/roundtrip-test.js b/js/tests/roundtrip-test.js
index 2c0859a..b81a736 100755
--- a/js/tests/roundtrip-test.js
+++ b/js/tests/roundtrip-test.js
@@ -2,13 +2,15 @@
"use strict";
var jsDiff = require( 'diff' ),
+ http = require( 'http' ),
optimist = require( 'optimist' ),
+ querystring = require( 'querystring' ),
domino = require( 'domino' ),
+ url = require( 'url' ),
zlib = require( 'zlib' ),
Util = require( '../lib/mediawiki.Util.js' ).Util,
DU = require( '../lib/mediawiki.DOMUtils.js' ).DOMUtils,
- WikitextSerializer = require(
'../lib/mediawiki.WikitextSerializer.js').WikitextSerializer,
TemplateRequest = require( '../lib/mediawiki.ApiRequest.js'
).TemplateRequest,
ParsoidConfig = require( '../lib/mediawiki.ParsoidConfig'
).ParsoidConfig,
MWParserEnvironment = require( '../lib/mediawiki.parser.environment.js'
).MWParserEnvironment;
@@ -380,8 +382,6 @@
};
var doubleRoundtripDiff = function ( env, offsets, body, out, cb ) {
- var src = env.page.src;
-
if ( offsets.length > 0 ) {
env.setPageSrcInfo( out );
env.errCB = function ( error ) {
@@ -390,7 +390,7 @@
};
var parserPipeline = Util.getParserPipeline( env,
'text/x-mediawiki/full' );
- parserPipeline.on( 'document', checkIfSignificant.bind( null,
env, offsets, src, body, out, cb ) );
+ parserPipeline.on( 'document', checkIfSignificant.bind( null,
env, offsets, env.page.src, body, out, cb ) );
parserPipeline.processToplevelDoc( out );
} else {
@@ -398,28 +398,71 @@
}
};
-var roundTripDiff = function ( env, document, cb ) {
- var out, diff, offsetPairs;
+var parsoidPost = function ( env, parsoidURL, prefix, title, text, oldid, cb )
{
+ var options = url.parse( url.resolve( parsoidURL, prefix + '/' + title
) );
+ options.method = 'POST';
+ options.headers = {
+ 'Content-Type': 'application/x-www-form-urlencoded',
+ };
- // Re-parse the HTML to uncover foster-parenting issues
- var origBody = document.body;
- document = domino.createDocument(DU.serializeNode(document));
+ var req = http.request( options, function( res ) {
+ res.setEncoding( 'utf8' );
+ res.body = '';
+
+ res.on( 'data', function ( chunk ) { res.body += chunk; } );
+
+ res.on( 'end', function () {
+ if ( res.statusCode !== 200 ) {
+ cb( res.body, null );
+ } else {
+ if ( env.profile ) {
+ // Record the time it's taken to parse
+ var timePrefix = oldid ? 'html2wt' :
'wt2html';
+ if ( res.headers[
'x-parsoid-performance' ] ) {
+ env.profile.time[ timePrefix ] =
+ parseInt( res.headers[
'x-parsoid-performance' ].
+ match(
/duration=((\d)+);/ )[1], 10 );
+ }
+ // Record the sizes
+ var sizePrefix = oldid ? 'wt' : 'html';
+ env.profile.size[ sizePrefix + 'raw' ] =
+ res.body.length;
+ // Compress to record the gzipped size
+ zlib.gzip( res.body, function( err,
gzippedbuf ) {
+ if ( !err ) {
+ env.profile.size[
sizePrefix + 'gzip' ] =
+
gzippedbuf.length;
+ }
+ cb( null, res );
+ } );
+ } else {
+ cb( null, res );
+ }
+ }
+ } );
+ } );
+
+ var data = {};
+ if ( oldid ) {
+ data.oldid = oldid;
+ data.html = text;
+ } else {
+ data.wt = text;
+ }
+ req.write( querystring.stringify( data ) );
+ req.end();
+};
+
+var roundTripDiff = function ( env, html, out, cb ) {
+ var diff, offsetPairs;
+
try {
- env.profile.time.serialize = new Date();
- out = new WikitextSerializer( { env: env }
).serializeDOM(document.body);
- env.profile.time.serialize = new Date() -
env.profile.time.serialize;
- env.profile.size.wikitext = out.length;
-
- // Finish the total time now
- if ( env.profile && env.profile.time ) {
- env.profile.time.total += new Date() -
env.profile.time.total_timer;
- delete( env.profile.time.total_timer );
- }
diff = jsDiff.diffLines( out, env.page.src );
offsetPairs = Util.convertDiffToOffsetPairs( diff );
if ( diff.length > 0 ) {
- doubleRoundtripDiff( env, offsetPairs, origBody, out,
cb );
+ var body = domino.createDocument( html ).body;
+ doubleRoundtripDiff( env, offsetPairs, body, out, cb );
} else {
cb( null, env, [] );
}
@@ -430,6 +473,11 @@
var fetch = function ( page, cb, options ) {
cb = typeof cb === 'function' ? cb : function () {};
+ var prefix = options.prefix || 'enwiki';
+
+ if ( options.apiURL ) {
+ prefix = 'customwiki';
+ }
var envCb = function ( err, env ) {
env.errCB = function ( error ) {
@@ -450,33 +498,31 @@
cb( err, env, [] );
} else {
env.setPageSrcInfo( src_and_metadata );
- env.profile.time.parse = new Date();
- Util.parse( env, function ( src, err, doc ) {
- env.profile.time.parse = new Date() -
env.profile.time.parse;
- if ( err ) {
- cb( err, env, [] );
- } else {
- // Pause the total time while
we compute these sizes
- env.profile.time.total += new
Date() - env.profile.time.total_timer;
- env.profile.size.htmlraw =
doc.outerHTML.length;
- zlib.gzip( doc.outerHTML,
function( err, buf ) {
- if ( !err ) {
-
env.profile.size.htmlgzip = buf.length;
- }
-
env.profile.time.total_timer = new Date();
- roundTripDiff( env,
doc, cb );
- });
- }
- }, err, env.page.src );
+ // First, fetch the HTML for the requested
page's wikitext
+ parsoidPost( env, options.parsoidURL, prefix,
page,
+ env.page.src, null, function ( err,
htmlRes ) {
+ if ( err ) {
+ cb( err, env, [] );
+ } else {
+ // And now, request the
wikitext for the obtained HTML
+ parsoidPost( env,
options.parsoidURL, prefix, page,
+ htmlRes.body,
src_and_metadata.revision.revid, function ( err, wtRes ) {
+ if (
err ) {
+
cb( err, env, [] );
+ } else {
+
// Finish the total time now
+
if ( env.profile && env.profile.time ) {
+
env.profile.time.total += new Date() - env.profile.time.total_timer;
+
delete( env.profile.time.total_timer );
+
}
+
roundTripDiff( env, htmlRes.body, wtRes.body, cb );
+ }
+ } );
+ }
+ } );
}
} );
};
-
- var prefix = options.prefix || 'enwiki';
-
- if ( options.apiURL ) {
- prefix = 'customwiki';
- }
var parsoidConfig = new ParsoidConfig( options, { defaultWiki: prefix }
);
@@ -501,6 +547,7 @@
process.exit( 1 );
} else {
console.log( output );
+ process.exit( 0 );
}
};
@@ -552,6 +599,10 @@
description: 'Dump state (see below for supported dump
flags)',
'boolean': false,
'default': ""
+ },
+ 'parsoidURL': {
+ description: 'The URL for the Parsoid API',
+ 'boolean': false
}
});
@@ -563,6 +614,13 @@
callback = cbCombinator.bind( null,
Util.booleanOption( argv.xml ) ?
xmlCallback : plainCallback,
consoleOut );
+ if ( !argv.parsoidURL ) {
+ // Start our own Parsoid server
+ // TODO: This will not be necessary once we have a
top-level testing
+ // script that takes care of setting everything up.
+ var apiServer = require( './apiServer.js' );
+ argv.parsoidURL = apiServer.startParsoidServer();
+ }
fetch( title, callback, argv );
} else {
opts.showHelp();
--
To view, visit https://gerrit.wikimedia.org/r/99348
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ibb9daa72d51d91c6192ac0d695e6ecc4e70005aa
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Parsoid
Gerrit-Branch: master
Gerrit-Owner: Marcoil <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
