Nikerabbit has uploaded a new change for review.
https://gerrit.wikimedia.org/r/170015
Change subject: Yandex backend
......................................................................
Yandex backend
Also enhanced error handling and reporting for the mt module.
Change-Id: I22f0802d8c435714625af789d317cb077d8ce02d
---
M ContentTranslationService.js
M config.defaults.js
M index.js
A mt/Yandex.js
M mt/index.js
5 files changed, 116 insertions(+), 9 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/cxserver
refs/changes/15/170015/1
diff --git a/ContentTranslationService.js b/ContentTranslationService.js
index 2719eb1..eb26ef0 100644
--- a/ContentTranslationService.js
+++ b/ContentTranslationService.js
@@ -91,7 +91,11 @@
);
} );
-app.post( '/mt/:from/:to/:provider?', function ( req, res ) {
+app.get( '/mt/:from/:to/:provider?', function ( req, res ) {
+ res.send( 405, { error: "Request must be posted" } );
+} );
+
+app.post( '/mt/:from/:to/:provider', function ( req, res ) {
var mtClients, mtClient,
sourceHtmlChunks, sourceHtml, reqLength,
registry = require( __dirname + '/registry' ),
@@ -100,13 +104,19 @@
provider = registry.getValidProvider( from, to, 'mt',
req.params.provider );
if ( !provider ) {
- res.send( 404 );
+ res.send( 404, { error: "Provider not supported" } );
logger.info( 'MT provider invalid or missing' );
return;
}
mtClients = require( __dirname + '/mt/' );
+ if ( mtClients[provider] === undefined ) {
+ res.send( 500, { error: "Provider not found" } );
+ logger.error( 'Configured provider ' + provider + ' was not
found' );
+ return;
+ }
+
mtClient = mtClients[ provider ];
sourceHtmlChunks = [ '<div>' ];
@@ -116,7 +126,7 @@
reqLength += data.length;
if ( reqLength > 50000 ) {
// Too long
- res.send( 500 );
+ res.send( 413, { error: "Content too long" } );
logger.error( 'MT content too long' );
return;
}
@@ -126,16 +136,14 @@
sourceHtmlChunks.push( '</div>' );
sourceHtml = sourceHtmlChunks.join( '' );
- logger.profile( 'MT');
+ logger.profile( 'MT' );
mtClient.translate( from, to, sourceHtml ).then(
function ( data ) {
res.send( data );
logger.profile( 'MT', { from: from, to: to } );
},
function ( error ) {
- res.send( 500, {
- error: error
- } );
+ res.send( 500, { error: error } );
logger.log( 'error', 'MT processing error:
(%s)', error.toString() );
}
);
diff --git a/config.defaults.js b/config.defaults.js
index 2422f8b..50989ae 100644
--- a/config.defaults.js
+++ b/config.defaults.js
@@ -9,6 +9,8 @@
'parsoid.api': 'http://parsoid-lb.eqiad.wikimedia.org',
// Apertium web API URL
'mt.apertium.api': 'http://apertium.wmflabs.org',
+ 'mt.yandex.api': 'https://translate.yandex.net',
+ 'mt.yandex.key': null,
// Use SSL?
secure: false,
// SSL key filename
@@ -433,6 +435,11 @@
providers: [
'Dictd'
]
+ },
+ mt: {
+ providers: [
+ "Yandex"
+ ]
}
},
sr: {
diff --git a/index.js b/index.js
index f955bdb..1b309e1 100644
--- a/index.js
+++ b/index.js
@@ -1,6 +1,5 @@
module.exports = {
Segmenter: require( './segmentation/CXSegmenter.js' ).CXSegmenter,
- Apertium: require( './mt/Apertium.js' ),
LinearDoc: require( './lineardoc/LinearDoc.js' ),
Dictionary: require( './dictionary' )
};
diff --git a/mt/Yandex.js b/mt/Yandex.js
new file mode 100644
index 0000000..1fb087f
--- /dev/null
+++ b/mt/Yandex.js
@@ -0,0 +1,92 @@
+var errormap,
+ Q = require( 'q' ),
+ request = require( 'request' ),
+ conf = require( __dirname + '/../utils/Conf.js' );
+
+// http://api.yandex.com/translate/doc/dg/reference/translate.xml
+errormap = {
+ 200: 'ERR_OK',
+ 401: 'ERR_KEY_INVALID',
+ 402: 'ERR_KEY_BLOCKED',
+ 403: 'ERR_DAILY_REQ_LIMIT_EXCEEDED',
+ 404: 'ERR_DAILY_CHAR_LIMIT_EXCEEDED',
+ 413: 'ERR_TEXT_TOO_LONG',
+ 422: 'ERR_UNPROCESSABLE_TEXT',
+ 501: 'ERR_LANG_NOT_SUPPORTED'
+}
+
+/**
+ * Returns error name from error code.
+ * @return {string}
+ */
+function getErrorName( code ) {
+ if ( code in errormap ) {
+ return errormap[code];
+ }
+
+ return 'Unknown error';
+}
+
+/**
+ * Translate plain text with Yandex.
+ *
+ * @param {string} sourceLang Source language code
+ * @param {string} targetLang Target language code
+ * @param {string} sourceText Source language text
+ * @return {Q.Promise} Target language text
+ */
+function translate( sourceLang, targetLang, sourceText ) {
+ var key, postData,
+ deferred = Q.defer();
+
+ key = conf( 'mt.yandex.key' );
+ if ( key === null ) {
+ deferred.reject( new Error( "Yandex service is misconfigured" )
);
+ return deferred.promise
+ }
+
+ if ( sourceText.length > 10000 ) {
+ deferred.reject( new Error( "Source text too long" ) );
+ return deferred.promise
+ }
+
+ // Language mapping that might be needed is be-tarask -> be
+ postData = {
+ url: conf( 'mt.yandex.api' ) + '/api/v1.5/tr.json/translate',
+ form: {
+ key: key,
+ lang: sourceLang + '-' + targetLang,
+ format: 'html',
+ text: sourceText
+ }
+ };
+
+ request.post( postData, function ( error, response, body ) {
+ var ret;
+
+ if ( error ) {
+ deferred.reject( new Error( error ) );
+ return;
+ }
+
+ try {
+ ret = JSON.parse( body );
+ } catch ( err ) {
+ deferred.reject( err );
+ return;
+ }
+
+ console.log( ret );
+ if ( ret.code !== 200 ) {
+ deferred.reject( new Error( ret.code + ": " +
getErrorName( code ) ) );
+ }
+
+ deferred.resolve( ret.text[0] );
+ } );
+
+ return deferred.promise;
+}
+
+module.exports = {
+ translate: translate,
+};
diff --git a/mt/index.js b/mt/index.js
index 6e6854b..acf5730 100644
--- a/mt/index.js
+++ b/mt/index.js
@@ -1,3 +1,4 @@
module.exports = {
- Apertium: require( './Apertium.js' )
+ Apertium: require( './Apertium.js' ),
+ Yandex: require( './Yandex.js' )
};
--
To view, visit https://gerrit.wikimedia.org/r/170015
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I22f0802d8c435714625af789d317cb077d8ce02d
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/cxserver
Gerrit-Branch: master
Gerrit-Owner: Nikerabbit <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
