jenkins-bot has submitted this change and it was merged. Change subject: Detect paste of wikitext and convert it ......................................................................
Detect paste of wikitext and convert it This invokes Parsoid to convert likely wikitext pastes. This addresses about 20% of the <nowiki> insertion quantified at https://fr.wikipedia.org/wiki/Wikip%C3%A9dia:%C3%89diteurVisuel/Avis/Nowiki for 2015-04-08 which were due to inadvertent pastes of wikitext. For a better user experience, suppress Parsoid's P-wrapping if the output consists of a single top-level P element. This ensures that pasting '[[Foo]]' won't add an unexpected newline before the paste. Bug: T54204 Bug: T109449 Change-Id: I26a4cd8dc5b7e7caf16ca081dbe7baf6a7db8e5c --- M VisualEditor.hooks.php M extension.json M modules/ve-mw/ce/nodes/ve.ce.MWTransclusionNode.js A modules/ve-mw/tests/ui/datatransferhandlers/ve.ui.MWWikitextStringTransferHandler.test.js A modules/ve-mw/ui/datatransferhandlers/ve.ui.MWWikitextStringTransferHandler.js 5 files changed, 358 insertions(+), 1 deletion(-) Approvals: Jforrester: Looks good to me, but someone else must approve Esanders: Looks good to me, approved jenkins-bot: Verified diff --git a/VisualEditor.hooks.php b/VisualEditor.hooks.php index 41cb6bf..1a79507 100644 --- a/VisualEditor.hooks.php +++ b/VisualEditor.hooks.php @@ -611,6 +611,7 @@ // VisualEditor DataTransferHandler tests 'lib/ve/tests/ui/datatransferhandlers/ve.ui.DSVFileTransferHandler.test.js', 'lib/ve/tests/ui/datatransferhandlers/ve.ui.UrlStringTransferHandler.test.js', + 'modules/ve-mw/tests/ui/datatransferhandlers/ve.ui.MWWikitextStringTransferHandler.test.js', // VisualEditor initialization Tests 'lib/ve/tests/init/ve.init.Platform.test.js', 'modules/ve-mw/tests/init/targets/ve.init.mw.DesktopArticleTarget.test.js', diff --git a/extension.json b/extension.json index e999934..183e459 100644 --- a/extension.json +++ b/extension.json @@ -950,6 +950,7 @@ "modules/ve-mw/ui/ve.ui.MWCommandRegistry.js", "modules/ve-mw/ui/ve.ui.MWSequenceRegistry.js", "modules/ve-mw/ui/commands/ve.ui.MWWikitextWarningCommand.js", + "modules/ve-mw/ui/datatransferhandlers/ve.ui.MWWikitextStringTransferHandler.js", "modules/ve-mw/ui/widgets/ve.ui.MWTargetWidget.js", "modules/ve-mw/ui/widgets/ve.ui.MWTocItemWidget.js", "modules/ve-mw/ui/widgets/ve.ui.MWTocWidget.js", diff --git a/modules/ve-mw/ce/nodes/ve.ce.MWTransclusionNode.js b/modules/ve-mw/ce/nodes/ve.ce.MWTransclusionNode.js index 966c624..fd2ac49 100644 --- a/modules/ve-mw/ce/nodes/ve.ce.MWTransclusionNode.js +++ b/modules/ve-mw/ce/nodes/ve.ce.MWTransclusionNode.js @@ -127,7 +127,7 @@ ve.ce.MWTransclusionNode.prototype.onParseSuccess = function ( deferred, response ) { var contentNodes; - if ( !response || response.error || !response.visualeditor || response.visualeditor.result !== 'success' ) { + if ( ve.getProp( response, 'visualeditor', 'result' ) !== 'success' ) { return this.onParseError.call( this, deferred ); } diff --git a/modules/ve-mw/tests/ui/datatransferhandlers/ve.ui.MWWikitextStringTransferHandler.test.js b/modules/ve-mw/tests/ui/datatransferhandlers/ve.ui.MWWikitextStringTransferHandler.test.js new file mode 100644 index 0000000..51801b1 --- /dev/null +++ b/modules/ve-mw/tests/ui/datatransferhandlers/ve.ui.MWWikitextStringTransferHandler.test.js @@ -0,0 +1,233 @@ +/*! + * VisualEditor UserInterface MWWikitextStringTransferHandler tests. + * + * @copyright 2011-2015 VisualEditor Team and others; see http://ve.mit-license.org + */ +var MWWIKITEXT_MOCK_API = true; + +QUnit.module( 've.ui.MWWikitextStringTransferHandler', QUnit.newMwEnvironment( { + setup: function () { + // Mock XHR for mw.Api() + this.server = MWWIKITEXT_MOCK_API ? this.sandbox.useFakeServer() : null; + } +} ) ); + +/* Tests */ + +function runWikitextStringHandlerTest( assert, server, string, mimeType, expectedResponse, expectedData, annotations, msg ) { + var handler, i, j, + done = assert.async(), + item = ve.ui.DataTransferItem.static.newFromString( string, mimeType ), + doc = ve.dm.example.createExampleDocument(), + mockSurface = { + getModel: function () { + return { + getDocument: function () { + return doc; + } + }; + } + }; + + // Preprocess the expectedData array + for ( i = 0; i < expectedData.length; i++ ) { + if ( Array.isArray( expectedData[ i ] ) ) { + for ( j = 0; j < expectedData[ i ][ 1 ].length; j++ ) { + if ( typeof expectedData[ i ][ 1 ][ j ] === 'number' ) { + expectedData[ i ][ 1 ][ j ] = annotations[ expectedData[ i ][ 1 ][ j ] ]; + } + } + } + } + + // Invoke the handler + handler = ve.ui.dataTransferHandlerFactory.create( 'wikitextString', mockSurface, item ); + + handler.getInsertableData().done( function ( doc2 ) { + var actualData = doc2.getData(); + ve.dm.example.postprocessAnnotations( actualData, doc2.getStore() ); + assert.equalLinearData( actualData, expectedData, msg + ': data match' ); + done(); + } ); + + if ( server ) { + server.respond( [ 200, { 'Content-Type': 'application/json' }, JSON.stringify( { + visualeditor: { + result: 'success', + content: expectedResponse + } + } ) ] ); + } +} + +QUnit.test( 'convert', function ( assert ) { + var i, + cases = [ + { + msg: 'Simple link', + pasteString: '[[Foo]]', + pasteType: 'text/plain', + parsoidResponse: '<body data-parsoid=\'{"dsr":[0,7,0,0]}\' lang="en" class="mw-content-ltr sitedir-ltr ltr mw-body mw-body-content mediawiki" dir="ltr"><p data-parsoid=\'{"dsr":[0,7,0,0]}\'><a rel="mw:WikiLink" href="./Foo" title="Foo" data-parsoid=\'{"stx":"simple","a":{"href":"./Foo"},"sa":{"href":"Foo"},"dsr":[0,7,2,2]}\'>Foo</a></p></body>', + annotations: [ { + type: 'link/mwInternal', + attributes: { + hrefPrefix: './', + lookupTitle: 'Foo', + normalizedTitle: 'Foo', + origTitle: 'Foo', + title: 'Foo' + } + } ], + expectedData: [ + [ 'F', [ 0 ] ], + [ 'o', [ 0 ] ], + [ 'o', [ 0 ] ], + { type: 'internalList' }, + { type: '/internalList' } + ] + }, + { + msg: 'Simple link with no p-wrapping', + pasteString: '*[[Foo]]', + pasteType: 'text/plain', + parsoidResponse: '<body data-parsoid=\'{"dsr":[0,8,0,0]}\' lang="en" class="mw-content-ltr sitedir-ltr ltr mw-body mw-body-content mediawiki" dir="ltr"><ul data-parsoid=\'{"dsr":[0,8,0,0]}\'><li data-parsoid=\'{"dsr":[0,8,1,0]}\'><a rel="mw:WikiLink" href="./Foo" title="Foo" data-parsoid=\'{"stx":"simple","a":{"href":"./Foo"},"sa":{"href":"Foo"},"dsr":[1,8,2,2]}\'>Foo</a></li></ul></body>', + annotations: [ { + type: 'link/mwInternal', + attributes: { + hrefPrefix: './', + lookupTitle: 'Foo', + normalizedTitle: 'Foo', + origTitle: 'Foo', + title: 'Foo' + } + } ], + expectedData: [ + { + type: 'list', + attributes: { style: 'bullet' } + }, + { type: 'listItem' }, + { + type: 'paragraph', + internal: { generated: 'wrapper' } + }, + [ 'F', [ 0 ] ], + [ 'o', [ 0 ] ], + [ 'o', [ 0 ] ], + { type: '/paragraph' }, + { type: '/listItem' }, + { type: '/list' }, + { type: 'internalList' }, + { type: '/internalList' } + ] + }, + { + msg: 'Heading', + pasteString: '==heading==', + pasteType: 'text/plain', + parsoidResponse: '<body data-parsoid=\'{"dsr":[0,11,0,0]}\' lang="en" class="mw-content-ltr sitedir-ltr ltr mw-body mw-body-content mediawiki" dir="ltr"><h2 data-parsoid=\'{"dsr":[0,11,2,2]}\'>heading</h2></body>', + annotations: [], + expectedData: [ + { type: 'heading', attributes: { level: 2 } }, + 'h', + 'e', + 'a', + 'd', + 'i', + 'n', + 'g', + { type: '/heading' }, + { type: 'internalList' }, + { type: '/internalList' } + ] + }, + { + msg: 'Magic link (RFC)', + pasteString: 'RFC 1234', + pasteType: 'text/plain', + parsoidResponse: '<body data-parsoid=\'{"dsr":[0,8,0,0]}\' lang="en" class="mw-content-ltr sitedir-ltr ltr mw-body mw-body-content mediawiki" dir="ltr"><p data-parsoid=\'{"dsr":[0,8,0,0]}\'><a href="//tools.ietf.org/html/rfc1234" rel="mw:ExtLink" data-parsoid=\'{"stx":"magiclink","dsr":[0,8,0,0]}\'>RFC 1234</a></p></body>', + annotations: [ { + type: 'link/mwExternal', + attributes: { + href: '//tools.ietf.org/html/rfc1234', + rel: 'mw:ExtLink' + } + } ], + expectedData: [ + [ 'R', [ 0 ] ], + [ 'F', [ 0 ] ], + [ 'C', [ 0 ] ], + [ ' ', [ 0 ] ], + [ '1', [ 0 ] ], + [ '2', [ 0 ] ], + [ '3', [ 0 ] ], + [ '4', [ 0 ] ], + { type: 'internalList' }, + { type: '/internalList' } + ] + }, + { + msg: 'Magic link (PMID)', + pasteString: 'PMID 1234', + pasteType: 'text/plain', + parsoidResponse: '<body data-parsoid=\'{"dsr":[0,9,0,0]}\' lang="en" class="mw-content-ltr sitedir-ltr ltr mw-body mw-body-content mediawiki" dir="ltr"><p data-parsoid=\'{"dsr":[0,9,0,0]}\'><a href="//www.ncbi.nlm.nih.gov/pubmed/1234?dopt=Abstract" rel="mw:ExtLink" data-parsoid=\'{"stx":"magiclink","dsr":[0,9,0,0]}\'>PMID 1234</a></p></body>', + annotations: [ { + type: 'link/mwExternal', + attributes: { + href: '//www.ncbi.nlm.nih.gov/pubmed/1234?dopt=Abstract', + rel: 'mw:ExtLink' + } + } ], + expectedData: [ + [ 'P', [ 0 ] ], + [ 'M', [ 0 ] ], + [ 'I', [ 0 ] ], + [ 'D', [ 0 ] ], + [ ' ', [ 0 ] ], + [ '1', [ 0 ] ], + [ '2', [ 0 ] ], + [ '3', [ 0 ] ], + [ '4', [ 0 ] ], + { type: 'internalList' }, + { type: '/internalList' } + ] + }, + { + msg: 'Magic link (ISBN)', + pasteString: 'ISBN 123456789X', + pasteType: 'text/plain', + parsoidResponse: '<body data-parsoid=\'{"dsr":[0,15,0,0]}\' lang="en" class="mw-content-ltr sitedir-ltr ltr mw-body mw-body-content mediawiki" dir="ltr"><p data-parsoid=\'{"dsr":[0,15,0,0]}\'><a href="./Special:BookSources/123456789X" rel="mw:ExtLink" data-parsoid=\'{"stx":"magiclink","dsr":[0,15,0,0]}\'>ISBN 123456789X</a></p></body>', + annotations: [ { + type: 'link/mwExternal', + attributes: { + href: './Special:BookSources/123456789X', + rel: 'mw:ExtLink' + } + } ], + expectedData: [ + [ 'I', [ 0 ] ], + [ 'S', [ 0 ] ], + [ 'B', [ 0 ] ], + [ 'N', [ 0 ] ], + [ ' ', [ 0 ] ], + [ '1', [ 0 ] ], + [ '2', [ 0 ] ], + [ '3', [ 0 ] ], + [ '4', [ 0 ] ], + [ '5', [ 0 ] ], + [ '6', [ 0 ] ], + [ '7', [ 0 ] ], + [ '8', [ 0 ] ], + [ '9', [ 0 ] ], + [ 'X', [ 0 ] ], + { type: 'internalList' }, + { type: '/internalList' } + ] + } + ]; + + QUnit.expect( cases.length ); + for ( i = 0; i < cases.length; i++ ) { + runWikitextStringHandlerTest( assert, this.server, cases[i].pasteString, cases[i].pasteType, cases[i].parsoidResponse, cases[i].expectedData, cases[i].annotations, cases[i].msg ); + } +} ); diff --git a/modules/ve-mw/ui/datatransferhandlers/ve.ui.MWWikitextStringTransferHandler.js b/modules/ve-mw/ui/datatransferhandlers/ve.ui.MWWikitextStringTransferHandler.js new file mode 100644 index 0000000..e520c2b --- /dev/null +++ b/modules/ve-mw/ui/datatransferhandlers/ve.ui.MWWikitextStringTransferHandler.js @@ -0,0 +1,122 @@ +/*! + * VisualEditor UserInterface MWWikitextStringTransferHandler class. + * + * @copyright 2011-2015 VisualEditor Team and others; see http://ve.mit-license.org + */ + +/** + * Detect an attempt to paste wikitext, and convert it to proper + * HTML. + * + * @class + * @extends ve.ui.PlainTextStringTransferHandler + * + * @constructor + * @param {ve.ui.Surface} surface + * @param {ve.ui.DataTransferItem} item + */ +ve.ui.MWWikitextStringTransferHandler = function VeUiMWWikitextStringTransferHandler() { + // Parent constructor + ve.ui.MWWikitextStringTransferHandler.super.apply( this, arguments ); +}; + +/* Inheritance */ + +OO.inheritClass( ve.ui.MWWikitextStringTransferHandler, ve.ui.PlainTextStringTransferHandler ); + +/* Static properties */ + +ve.ui.MWWikitextStringTransferHandler.static.name = 'wikitextString'; + +ve.ui.MWWikitextStringTransferHandler.static.types = + ve.ui.MWWikitextStringTransferHandler.super.static.types.concat( + [ 'text/x-wiki' ] + ); + +ve.ui.MWWikitextStringTransferHandler.static.handlesPaste = true; + +/** + * Heuristic pattern which attempts to discover wikitext, without + * incurring too many false positives. + * + * Currently the pattern looks for ==...==, [[...]], or {{...}} + * which occur on a single line of max 80 characters. + */ +ve.ui.MWWikitextStringTransferHandler.static.matchRegExp = + /(^\s*(={2,6})[^=\r\n]{1,80}\1\s*$)|\[\[.{1,80}\]\]|\{\{.{1,80}\}\}/m; + +ve.ui.MWWikitextStringTransferHandler.static.matchFunction = function ( item ) { + var text = item.getAsString(); + + // If the mime type is explicitly wikitext (ie, not plain text), + // always accept. + if ( item.type === 'text/x-wiki' ) { + return true; + } + + // Detect autolink opportunities for magic words. + // (The link should be the only contents of paste to match this heuristic) + if ( /^\s*(RFC|ISBN|PMID)[-\s0-9]+[Xx]?\s*$/.test( text ) ) { + return true; + } + + // Use a heuristic regexp to find text likely to be wikitext. + // This test could be made more sophisticated in the future. + if ( this.matchRegExp.test( text ) ) { + return true; + } + return false; +}; + +/* Methods */ + +/** + * @inheritdoc + */ +ve.ui.MWWikitextStringTransferHandler.prototype.process = function () { + var xhr, + handler = this, + wikitext = this.item.getAsString(); + + function failure() { + // There's no DTH fallback handling for failures, so just paste + // the raw wikitext if things go wrong. + handler.resolve( wikitext ); + } + + // Convert wikitext to html using Parsoid. + xhr = new mw.Api().post( { + action: 'visualeditor', + paction: 'parsefragment', + page: mw.config.get( 'wgRelevantPageName' ), + wikitext: wikitext + } ).then( function ( response ) { + var doc, surface; + if ( ve.getProp( response, 'visualeditor', 'result' ) !== 'success' ) { + return failure(); + } + + doc = handler.surface.getModel().getDocument().newFromHtml( + response.visualeditor.content, + null // No sanitization, since HTML is from Parsoid + ); + + // Attempt to undo outermost p-wrapping if possible + surface = new ve.dm.Surface( doc ); + try { + surface.change( + ve.dm.Transaction.newFromWrap( doc, new ve.Range( 0, doc.data.countNonInternalElements() ), [], [], [ { type: 'paragraph' } ], [] ) + ); + } catch ( e ) { + // Sometimes there is no p-wrapping, for example: "* foo" + // Sometimes there are multiple <p> tags in the output. + // That's okay: ignore the error and paste what we've got. + } + + handler.resolve( doc ); + }, failure ); +}; + +/* Registration */ + +ve.ui.dataTransferHandlerFactory.register( ve.ui.MWWikitextStringTransferHandler ); -- To view, visit https://gerrit.wikimedia.org/r/230146 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I26a4cd8dc5b7e7caf16ca081dbe7baf6a7db8e5c Gerrit-PatchSet: 19 Gerrit-Project: mediawiki/extensions/VisualEditor Gerrit-Branch: master Gerrit-Owner: Cscott <[email protected]> Gerrit-Reviewer: Bartosz Dziewoński <[email protected]> Gerrit-Reviewer: Cscott <[email protected]> Gerrit-Reviewer: Eranroz <[email protected]> Gerrit-Reviewer: Esanders <[email protected]> Gerrit-Reviewer: Jforrester <[email protected]> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
