Santhosh has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/381418 )
Change subject: Template adaptation: Implement template param name matching ...................................................................... Template adaptation: Implement template param name matching Algorithm: 1. Get template data for both source and target 2. Get aliases array for each parameter in source template, also add the param name to that and form aliases+name set 3. For each param in target template data, form a similar set of aliases and param. 4. Find intersection of above two sets. 5. If the intersection set is not empty, the source param maps to the param name we located in target template data Tests for name matching added. TODO: 1. Use the param mapping to build target template definition. 2. Update tests for MWTemplate.js Bug: T162114 Change-Id: I4ea9ae0dc902fecba0d300864e40cae6595c61b6 --- D config.yaml A config.yaml A lib/adaptation/TemplateParameterMapper.js M lib/adaptation/TemplateTransclusion.js A test/adaptation/TemplateParameterMapper.test.js M test/translationunits/MWTemplate.test.json 6 files changed, 357 insertions(+), 49 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/cxserver refs/changes/18/381418/1 diff --git a/config.yaml b/config.yaml deleted file mode 120000 index c11eec8..0000000 --- a/config.yaml +++ /dev/null @@ -1 +0,0 @@ -config.dev.yaml \ No newline at end of file diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..36280af --- /dev/null +++ b/config.yaml @@ -0,0 +1,93 @@ +# Number of worker processes to spawn. +# Set to 0 to run everything in a single process without clustering. +# Use ncpu to run as many workers as there are CPU units +num_workers: 0 + +# Log error messages and gracefully restart a worker if v8 reports that it +# uses more heap (note: not RSS) than this many megabytes. +worker_heap_limit_mb: 250 + +# Logger info +logging: + level: trace +# streams: +# # Use gelf-stream -> logstash +# - type: gelf +# host: logstash1003.eqiad.wmnet +# port: 12201 + +# Statsd metrics reporter +metrics: + type: log + #host: localhost + #port: 8125 + +services: + - name: cxserver + # a relative path or the name of an npm package, if different from name + module: ./app.js + # optionally, a version constraint of the npm package + # version: ^0.4.0 + # per-service config + conf: + port: 8080 + # interface: localhost # uncomment to only listen on localhost + # More per-service config settings + # The location of the spec, defaults to spec.yaml if not specified + # spec: ./spec.yaml + # allow cross-domain requests to the API (default *) + cors: '*' + # to disable use: + # cors: false + # to restrict to a particular domain, use: + # cors: restricted.domain.org + # URL of the outbound proxy to use (complete with protocol) + # proxy: http://my.proxy.org:8080 + # the list of domains for which not to use the proxy defined above + # no_proxy_list: + # - domain1.com + # - domain2.org + user_agent: cxserver + # Mediawiki host name. Example {lang}.wikisource.org which get expanded internally to + # es.wikisource.org in a spanish language context. + # Do not prefix with http or https:// + # mw_host: '{lang}.wikipedia.org' + mwapi_req: + body: "{{request.body}}" + query: "{{ default(request.query, {}) }}" + headers: + host: "{{request.params.domain}}" + user-agent: "{{user-agent}}" + method: post + uri: "https://{{domain}}/w/api.php"; + restbase_req: + method: '{{request.method}}' + uri: https://{{domain}}/api/rest_v1/{+path} + query: '{{ default(request.query, {}) }}' + headers: '{{request.headers}}' + body: '{{request.body}}' + jwt: + secret: 'secret' + algorithms: + - HS256 + languages: config/languages.yaml + mt: + Apertium: + api: http://apertium.wmflabs.org + languages: config/Apertium.yaml + Yandex: + api: https://translate.yandex.net + key: null + languages: config/Yandex.yaml + Youdao: + api: https://fanyi.youdao.com/paidapi/fanyiapi + key: null + languages: config/Youdao.yaml + Matxin: + api: http://matxin.elhuyar.eus/API + languages: config/Matxin.yaml + dictionary: + Dictd: + languages: config/Dictd.yaml + JsonDict: + languages: config/JsonDict.yaml diff --git a/lib/adaptation/TemplateParameterMapper.js b/lib/adaptation/TemplateParameterMapper.js new file mode 100644 index 0000000..cb9a1f0 --- /dev/null +++ b/lib/adaptation/TemplateParameterMapper.js @@ -0,0 +1,59 @@ +class TemplateParameterMapper { + constructor( sourceParams, sourceTemplateData, targetTemplateData ) { + this.sourceParams = sourceParams; + this.sourceTemplateData = sourceTemplateData; + this.targetTemplateData = targetTemplateData; + this.parameterMap = {}; + } + + getAdaptedParameters() { + // TODO: Use parameter map to copy values of sourceParams + return this.sourceParams; + } + + getParameterMap() { + + for ( let name in this.sourceParams ) { + let normalizedKey = name.trim().toLowerCase().replace( /[\s+_-]+/g, '' ); + + if ( !isNaN( name ) ) { + // name is like "1" or "2" etc. Unnamed params. + this.parameterMap[ name ] = name; + continue; + } + + if ( !this.targetTemplateData.params ) { + // No params in target template data definition + continue; + } + + let sourceAiases = ( this.sourceTemplateData.params[ name ] || this.sourceTemplateData.params[ normalizedKey ] ).aliases; + let sourceKeyAndAliases = new Set( sourceAiases ); + sourceKeyAndAliases.add( name ); + sourceKeyAndAliases.add( normalizedKey ); + + // Search in the aliases for a match - case insensitive. + for ( let paramName in this.targetTemplateData.params ) { + const param = this.targetTemplateData.params[ paramName ]; + + let targetKeyAndAliases = new Set( param.aliases ); + targetKeyAndAliases.add( paramName ); + + // Find intersection of sourceKeyAndAliases and targetKeyAndAliases + let intersection = new Set( [ ...targetKeyAndAliases ].filter( key => { + let normalizedKey = key.trim().toLowerCase().replace( /[\s+_-]+/g, '' ); + return sourceKeyAndAliases.has( normalizedKey ); + } ) ); + + if ( intersection.size > 0 ) { + // Found a match + this.parameterMap[ name ] = paramName; + break; + } + } + } + return this.parameterMap; + } +} + +module.exports = TemplateParameterMapper; diff --git a/lib/adaptation/TemplateTransclusion.js b/lib/adaptation/TemplateTransclusion.js index 5e26158..8cb9a01 100644 --- a/lib/adaptation/TemplateTransclusion.js +++ b/lib/adaptation/TemplateTransclusion.js @@ -1,10 +1,11 @@ 'use strict'; const cxutil = require( '../util' ); +const TemplateParameterMapper = require( './TemplateParameterMapper' ); class TemplateTransclusion { - constructor( part, api, sourceLanguage, targetLanguage ) { - this.in = part; + constructor( transclusionDef, api, sourceLanguage, targetLanguage ) { + this.transclusionDef = transclusionDef; this.targetNameLookup = ( name ) => api.titlePairRequest( name, sourceLanguage, targetLanguage ); this.sourceTemplateDataLookup = ( name ) => api.templateDataRequest( name, sourceLanguage ); @@ -14,11 +15,11 @@ } TemplateTransclusion.prototype.adapt = cxutil.async( function* () { - let out = this.in; + let adapedTransclusionDef = this.transclusionDef; let metadata = {}; - const sourceName = this.in.template.target.href; - const targetInfo = yield this.targetNameLookup( sourceName ); + const sourceTitle = this.transclusionDef.template.target.href; + const targetInfo = yield this.targetNameLookup( sourceTitle ); const targetTitle = targetInfo.targetTitle; if ( !targetTitle ) { // Return the template unadapted, but let the client know it is not adaptable. @@ -28,24 +29,22 @@ const templateNamespaceAlias = yield this.targetNamespaceLookup(); // E.g. `Babel` for `Template:Babel` but `User:Babel` for `User:Babel` const targetTemplate = targetTitle.replace( templateNamespaceAlias + ':', '' ); - out.template.target.href = './' + targetTitle; - out.template.target.wt = targetTemplate; + adapedTransclusionDef.template.target.href = './' + targetTitle; + adapedTransclusionDef.template.target.wt = targetTemplate; - // TODO: Handle parameters - // const sourceParams = this.in.params; - // mapper = new TemplateParameterMapper( sourceParams, templateDataForSource, templateDataForTarget ); - // out.params = mapper.getAdaptedParameters(); - // metadata.parameterMap = mapper.getParameterMap(); - - // const sourceTemplateData = yield this.sourceTemplateDataLookup( sourceName ); - // const targetTemplateData = yield this.targetTemplateDataLookup( targetName ); - // metadata.templateData = { source: sourceTemplateData, target: targetTemplateData } + const sourceParams = this.transclusionDef.template.params; + const sourceTemplateData = yield this.sourceTemplateDataLookup( sourceTitle ); + const targetTemplateData = yield this.targetTemplateDataLookup( targetTitle ); + const mapper = new TemplateParameterMapper( sourceParams, sourceTemplateData, targetTemplateData ); + adapedTransclusionDef.template.params = mapper.getAdaptedParameters(); + metadata.parameterMap = mapper.getParameterMap(); + metadata.templateData = { source: sourceTemplateData, target: targetTemplateData }; metadata.adapted = true; } return { - formatForParsoid: () => out, + formatForParsoid: () => adapedTransclusionDef, getMetadata: () => metadata }; } ); diff --git a/test/adaptation/TemplateParameterMapper.test.js b/test/adaptation/TemplateParameterMapper.test.js new file mode 100644 index 0000000..69d0905 --- /dev/null +++ b/test/adaptation/TemplateParameterMapper.test.js @@ -0,0 +1,64 @@ +'use strict'; + +const assert = require( '../utils/assert.js' ), + TemplateParameterMapper = require( '../../lib/adaptation/TemplateParameterMapper' ); + +const test = { + sourceParams: { + 0: { wt: 'FirstValue' }, + name: { wt: 'NameValue' }, + firstname: { wt: 'FirstNameValue' }, + 'namewithatailingspace ': { wt: 'SomeValue' }, + author: { wt: 'AuthorName' }, + 'author-link1': { wt: 'author-link1-value' } + }, + sourceTemplateData: { + params: { + name: { + aliases: [ 'srcName' ] + }, + firstname: { aliases: [ 'first-name' ] }, + 'namewithatailingspace ': {}, + author: { aliases: [ 'writer' ] }, + 'author-link1': { aliases: [ 'authorlink' ] } + } + }, + targetTemplateData: { + params: { + título: { + aliases: [ 'Name', 'nombre' ] + }, + 'primero-título': { + aliases: [ 'First_Name', 'naciente' ] + }, + // eslint-disable-next-line camelcase + name_with_a_tailing_space: {}, + escritor: { + aliases: [ 'Writer' ] + }, + 'escritor-link': { + aliases: [ 'authorlink' ] + } + } + }, + expectedParamMapping: { + // Unnamed param name + 0: '0', + // Mapped by alises of target + name: 'título', + // Mapped by alises of target + firstname: 'primero-título', + // Normalized source name, target name match + 'namewithatailingspace ': 'name_with_a_tailing_space', + // Mapped by alises of source and target + author: 'escritor', + // Mapped by alises of source and target + 'author-link1': 'escritor-link' + } +}; +describe( 'Template Param mapping tests', () => { + const mapper = new TemplateParameterMapper( test.sourceParams, test.sourceTemplateData, test.targetTemplateData ); + it( 'should not have any errors while mapping params', () => { + assert.deepEqual( mapper.getParameterMap(), ( test.expectedParamMapping ) ); + } ); +} ); diff --git a/test/translationunits/MWTemplate.test.json b/test/translationunits/MWTemplate.test.json index aacc5e2..06d0a68 100644 --- a/test/translationunits/MWTemplate.test.json +++ b/test/translationunits/MWTemplate.test.json @@ -1,58 +1,152 @@ [ { - "desc":"Babel template", - "from":"fi", - "to":"sl", - "source":{ - "attributes":{ - "id":"mwSQ", - "typeof":"mw:Transclusion", - "data-mw":{ - "parts":[ + "desc": "Babel template", + "from": "fi", + "to": "sl", + "source": { + "attributes": { + "id": "mwSQ", + "typeof": "mw:Transclusion", + "data-mw": { + "parts": [ { - "template":{ - "target":{ - "wt":"Babel", - "href":"./Malline:Babel" + "template": { + "target": { + "wt": "Babel", + "href": "./Malline:Babel" }, - "params":{ - "1":{ - "wt":"fi" + "params": { + "1": { + "wt": "fi" } }, - "i":0 + "i": 0 } } ] } } }, - "result":{ - "attributes":{ - "data-cx":[ + "result": { + "attributes": { + "data-cx": [ { - "adapted":true + "adapted": true } ], - "data-mw":{ - "parts":[ + "data-mw": { + "parts": [ { - "template":{ - "target":{ - "wt":"Babilon", - "href":"./Predloga:Babilon" + "template": { + "target": { + "wt": "Babilon", + "href": "./Predloga:Babilon" }, - "params":{ - "1":{ - "wt":"fi" + "params": { + "1": { + "wt": "fi" } }, - "i":0 + "i": 0 } } ] } } } + }, + { + "desc": "Cite web template", + "from": "en", + "to": "fr", + "source": { + "attributes": { + "id": "mwSQ", + "typeof": "mw:Transclusion", + "data-mw": { + "parts": [{ + "template": { + "target": { + "wt": "cite web ", + "href": "./Template:Cite_web " + }, + "params": { + "url": { + "wt": "http://www.example.org/ " + }, + "title": { + "wt": "My Favorite Things, Part II " + }, + "last": { + "wt": "Doe " + }, + "first": { + "wt": "John " + }, + "publisher": { + "wt": " Open Publishing " + }, + "date": { + "wt": "30 April 2005 " + }, + "website": { + "wt": "Encyclopedia of Things " + }, + "access-date ": { + "wt": "26 June 2016 " + } + }, + "i": 0 + } + }] + } + } + }, + "result": { + "attributes": { + "data-cx": [ + { + "adapted": true + } + ], + "data-mw": { + "parts": [{ + "template": { + "target": { + "wt": "Lien web ", + "href": "./Modèle:Lien web" + }, + "params": { + "url": { + "wt": "http://www.example.org/ " + }, + "title": { + "wt": "My Favorite Things, Part II " + }, + "last": { + "wt": "Doe " + }, + "first": { + "wt": "John " + }, + "publisher": { + "wt": " Open Publishing " + }, + "date": { + "wt": "30 April 2005 " + }, + "website": { + "wt": "Encyclopedia of Things " + }, + "access-date ": { + "wt": "26 June 2016 " + } + }, + "i": 0 + } + }] + } + } + } } ] -- To view, visit https://gerrit.wikimedia.org/r/381418 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I4ea9ae0dc902fecba0d300864e40cae6595c61b6 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/services/cxserver Gerrit-Branch: master Gerrit-Owner: Santhosh <santhosh.thottin...@gmail.com> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
