jenkins-bot has submitted this change and it was merged.
Change subject: Support for CLDR plural keywords in JSON file format
......................................................................
Support for CLDR plural keywords in JSON file format
This adds a configuration flag, parseCLDRPlurals, for the JSON
format. If set to yes, we will flatten/unflatten plural keywords
in the JSON source similarly to how it's already done for YAML,
using largely identical code (but also supporting the separator
option).
This makes redundant the YAML subclass implementation, which will
be refactored out in a follow-up change.
Change-Id: I15a9fb4a7f01f7ddb2f0b64c3a81062219dc2ac9
---
M ffs/JsonFFS.php
M tests/phpunit/utils/ArrayFlattenerTest.php
M utils/ArrayFlattener.php
3 files changed, 333 insertions(+), 15 deletions(-)
Approvals:
Nikerabbit: Looks good to me, approved
jenkins-bot: Verified
diff --git a/ffs/JsonFFS.php b/ffs/JsonFFS.php
index 9e89d19..b94a7ba 100644
--- a/ffs/JsonFFS.php
+++ b/ffs/JsonFFS.php
@@ -49,8 +49,11 @@
unset( $messages['@metadata'] );
if ( isset( $this->extra['nestingSeparator'] ) ) {
- $flattener = new ArrayFlattener(
$this->extra['nestingSeparator'] );
- $messages = $flattener->flatten( $messages );
+ $parseCLDRPlurals = isset(
$this->extra['parseCLDRPlurals'] ) ?
+ $this->extra['parseCLDRPlurals'] : false;
+ $flattener = new ArrayFlattener(
$this->extra['nestingSeparator'],
+ $parseCLDRPlurals );
+ $messages = $flattener->flatten( $messages );
}
$messages = $this->group->getMangler()->mangle( $messages );
@@ -111,8 +114,11 @@
}
if ( isset( $this->extra['nestingSeparator'] ) ) {
- $flattener = new ArrayFlattener(
$this->extra['nestingSeparator'] );
- $messages = $flattener->unflatten( $messages );
+ $parseCLDRPlurals = isset(
$this->extra['parseCLDRPlurals'] ) ?
+ $this->extra['parseCLDRPlurals'] : false;
+ $flattener = new ArrayFlattener(
$this->extra['nestingSeparator'],
+ $parseCLDRPlurals );
+ $messages = $flattener->unflatten( $messages );
}
return FormatJson::encode( $messages, "\t", FormatJson::ALL_OK
) . "\n";
@@ -129,6 +135,9 @@
'nestingSeparator' =>
array(
'_type' =>
'text',
),
+ 'parseCLDRPlurals' =>
array(
+ '_type' =>
'boolean',
+ )
)
)
)
diff --git a/tests/phpunit/utils/ArrayFlattenerTest.php
b/tests/phpunit/utils/ArrayFlattenerTest.php
index 9a55d4d..84984ff 100644
--- a/tests/phpunit/utils/ArrayFlattenerTest.php
+++ b/tests/phpunit/utils/ArrayFlattenerTest.php
@@ -26,6 +26,34 @@
$this->assertEquals( $expected, $output );
}
+ /**
+ * @dataProvider provideTestCLDRPlurals
+ */
+ public function testFlattenCLDRPlurals( $sep, $input, $expected ) {
+ $flattener = new ArrayFlattener( $sep, true );
+ $output = $flattener->flatten( $input );
+ $this->assertEquals( $expected, $output );
+ }
+
+ /**
+ * @dataProvider provideTestCLDRPlurals
+ */
+ public function testUnflattenCLDRPlurals( $sep, $expected, $input ) {
+ $flattener = new ArrayFlattener( $sep, true );
+ $output = $flattener->unflatten( $input );
+ $this->assertEquals( $expected, $output );
+ }
+
+ /**
+ * @expectedException MWException
+ * @expectedExceptionMessage Reserved plural keywords mixed with other
keys
+ * @dataProvider provideTestMixedCLDRPlurals
+ */
+ public function testFlattenMixedCLDRPlurals( $input ) {
+ $flattener = new ArrayFlattener( '.', true );
+ $flattener->flatten( $input );
+ }
+
public static function provideTestFlatten() {
$cases = array();
$cases[] = array(
@@ -40,6 +68,99 @@
array( 'a.b.c' => 1, 'a.b.d' => 2 ),
);
+ // By default, CLDR plural keywords should be treated like any
other key
+ $cases[] = array(
+ '/',
+ array( 'number' => array( 'one' => '1', 'other' =>
'999' ) ),
+ array( 'number/one' => '1', 'number/other' => '999' )
+ );
+
return $cases;
}
+
+ public static function provideTestCLDRPlurals() {
+ $cases = array();
+
+ // We include some non-plural data to ensure it is processed
correctly
+ $cases[] = array(
+ '/',
+ array(
+ 'cat' => 'An amount of cats',
+ 'mice' => array(
+ 'Frankie',
+ 'Benjy'
+ ),
+ 'dog or dogs' => array(
+ 'one' => 'One dog',
+ 'two' => 'Two doggies',
+ 'other' => 'Some dogs'
+ )
+ ),
+ array(
+ 'cat' => 'An amount of cats',
+ 'mice/0' => 'Frankie',
+ 'mice/1' => 'Benjy',
+ 'dog or dogs' => '{{PLURAL|one=One dog|two=Two
doggies|Some dogs}}'
+ ),
+ );
+
+ $cases[] = array(
+ '/',
+ array(
+ 'dog or dogs' => array(
+ 'zero' => 'No dogs',
+ 'one' => 'One dog',
+ 'two' => 'A couple doggies',
+ 'few' => 'A few dogs',
+ 'many' => '%1 dogs',
+ 'other' => 'Some dogs'
+ )
+ ),
+ array(
+ 'dog or dogs' => '{{PLURAL|zero=No dogs|one=One
dog|two=A couple doggies|' .
+ 'few=A few dogs|many=%1 dogs|Some
dogs}}'
+ ),
+ );
+
+ $cases[] = array(
+ '/',
+ array(
+ 'math is hard' => array(
+ 'one' => 'a=400',
+ 'other' => 'a=999'
+ )
+ ),
+ array( 'math is hard' => '{{PLURAL|one=a=400|a=999}}' ),
+ );
+
+ return $cases;
+ }
+
+ // Separate provider because the input throws an exception
+ public static function provideTestMixedCLDRPlurals() {
+ $cases = array();
+ $cases[] = array(
+ array(
+ 'dog or dogs' => array(
+ 'one' => 'One dog',
+ 'two' => 'Two doggies',
+ 'other' => 'Some dogs',
+ 'Pluto' => 'A specific dog'
+ )
+ )
+ );
+
+ $cases[] = array(
+ array(
+ 'dog or dogs' => array(
+ 'Pluto' => 'A specific dog',
+ 'one' => 'One dog',
+ 'two' => 'Two doggies',
+ 'other' => 'Some dogs',
+ )
+ )
+ );
+ return $cases;
+ }
+
}
diff --git a/utils/ArrayFlattener.php b/utils/ArrayFlattener.php
index 8845ef1..c65081c 100644
--- a/utils/ArrayFlattener.php
+++ b/utils/ArrayFlattener.php
@@ -1,9 +1,12 @@
<?php
/**
- * Support for JSON message file format.
+ * Flattens message arrays for further processing. Supports parsing CLDR
+ * plural messages and converting them into MediaWiki's {{PLURAL}} syntax
+ * in a single message.
*
* @file
* @author Niklas Laxström
+ * @author Erik Moeller
* @license GPL-2.0+
* @since 2016.01
*/
@@ -11,14 +14,25 @@
class ArrayFlattener {
protected $sep;
- public function __construct( $sep = '.' ) {
+ // For CLDR pluralization rules
+ protected static $pluralWords = array(
+ 'zero' => 1,
+ 'one' => 1,
+ 'many' => 1,
+ 'few' => 1,
+ 'other' => 1,
+ 'two' => 1
+ );
+
+ public function __construct( $sep = '.', $parseCLDRPlurals = false ) {
$this->sep = $sep;
+ $this->parseCLDRPlurals = $parseCLDRPlurals;
}
/**
* Flattens multidimensional array.
*
- * @param array $unflat It's an array.
+ * @param array $unflat Array of messages
* @return array
*/
public function flatten( array $unflat ) {
@@ -30,14 +44,21 @@
continue;
}
- // Placeholder for special plural processing
-
- $temp = array();
- foreach ( $value as $subKey => $subValue ) {
- $newKey = "$key{$this->sep}$subKey";
- $temp[$newKey] = $subValue;
+ $plurals = false;
+ if ( $this->parseCLDRPlurals ) {
+ $plurals = $this->flattenCLDRPlurals( $value );
}
- $flat += $this->flatten( $temp );
+
+ if ( $this->parseCLDRPlurals && $plurals ) {
+ $flat[$key] = $plurals;
+ } else {
+ $temp = array();
+ foreach ( $value as $subKey => $subValue ) {
+ $newKey = "$key{$this->sep}$subKey";
+ $temp[$newKey] = $subValue;
+ }
+ $flat += $this->flatten( $temp );
+ }
// Can as well keep only one copy around.
unset( $unflat[$key] );
@@ -47,15 +68,87 @@
}
/**
+ * Flattens arrays that contain CLDR plural keywords into single values
using
+ * MediaWiki's plural syntax.
+ *
+ * @param array $messages Array of messages
+ *
+ * @throws MWException
+ * @return bool|string
+ */
+ public function flattenCLDRPlurals( $messages ) {
+
+ $pluralKeys = false;
+ $nonPluralKeys = false;
+ foreach ( $messages as $key => $value ) {
+ if ( is_array( $value ) ) {
+ // Plurals can only happen in the lowest level
of the structure
+ return false;
+ }
+
+ // Check if we find any reserved plural keyword
+ if ( isset( self::$pluralWords[$key] ) ) {
+ $pluralKeys = true;
+ } else {
+ $nonPluralKeys = true;
+ }
+ }
+
+ // No plural keys at all, we can skip
+ if ( !$pluralKeys ) {
+ return false;
+ }
+
+ // Mixed plural keys with other keys, should not happen
+ if ( $nonPluralKeys ) {
+ $keys = implode( ', ', array_keys( $messages ) );
+ throw new MWException( "Reserved plural keywords mixed
with other keys: $keys." );
+ }
+
+ $pls = '{{PLURAL';
+ foreach ( $messages as $key => $value ) {
+ if ( $key === 'other' ) {
+ continue;
+ }
+
+ $pls .= "|$key=$value";
+ }
+
+ // Put the "other" alternative last, without other= prefix.
+ $other = isset( $messages['other'] ) ? '|' . $messages['other']
: '';
+ $pls .= "$other}}";
+
+ return $pls;
+ }
+
+ /**
* Performs the reverse operation of flatten.
*
- * @param array $flat It's an array
+ * @param array $flat Array of messages
* @return array
*/
public function unflatten( $flat ) {
+
$unflat = array();
+ if ( $this->parseCLDRPlurals ) {
+ $unflattenedPlurals = array();
+ foreach ( $flat as $key => $value ) {
+ $plurals = false;
+ if ( !is_array( $value ) ) {
+ $plurals = $this->unflattenCLDRPlurals(
$key, $value );
+ }
+ if ( $plurals ) {
+ $unflattenedPlurals += $plurals;
+ } else {
+ $unflattenedPlurals[$key] = $value;
+ }
+ }
+ $flat = $unflattenedPlurals;
+ }
+
foreach ( $flat as $key => $value ) {
+
$path = explode( $this->sep, $key );
if ( count( $path ) === 1 ) {
$unflat[$key] = $value;
@@ -86,4 +179,99 @@
return $unflat;
}
+
+ /**
+ * Converts the MediaWiki plural syntax to array of CLDR style plurals
+ *
+ * @param string $key Message key prefix
+ * @param string $message The plural string
+ *
+ * @return bool|array
+ */
+ public function unflattenCLDRPlurals( $key, $message ) {
+
+ // Quick escape.
+ if ( strpos( $message, '{{PLURAL' ) === false ) {
+ return false;
+ }
+
+ /*
+ * Replace all variables with placeholders. Possible source of
bugs
+ * if other characters that given below are used.
+ */
+ $regex = '~\{[a-zA-Z_-]+}~';
+ $placeholders = array();
+ $match = array();
+
+ while ( preg_match( $regex, $message, $match ) ) {
+ $uniqkey = TranslateUtils::getPlaceholder();
+ $placeholders[$uniqkey] = $match[0];
+ $search = preg_quote( $match[0], '~' );
+ $message = preg_replace( "~$search~", $uniqkey,
$message );
+ }
+
+ // Then replace (possible multiple) plural instances into
placeholders.
+ $regex = '~\{\{PLURAL\|(.*?)}}~s';
+ $matches = array();
+ $match = array();
+
+ while ( preg_match( $regex, $message, $match ) ) {
+ $uniqkey = TranslateUtils::getPlaceholder();
+ $matches[$uniqkey] = $match;
+ $message = preg_replace( $regex, $uniqkey, $message, 1
);
+ }
+
+ // No plurals, should not happen.
+ if ( !count( $matches ) ) {
+ return false;
+ }
+
+ // The final array of alternative plurals forms.
+ $alts = array();
+
+ /*
+ * Then loop trough each plural block and replacing the
placeholders
+ * to construct the alternatives. Produces invalid output if
there is
+ * multiple plural bocks which don't have the same set of keys.
+ */
+ $pluralChoice = implode( '|', array_keys( self::$pluralWords )
);
+ $regex = "~($pluralChoice)\s*=\s*(.+)~s";
+ foreach ( $matches as $ph => $plu ) {
+ $forms = explode( '|', $plu[1] );
+
+ foreach ( $forms as $form ) {
+ if ( $form === '' ) {
+ continue;
+ }
+
+ $match = array();
+ if ( preg_match( $regex, $form, $match ) ) {
+ $formWord =
"$key{$this->sep}{$match[1]}";
+ $value = $match[2];
+ } else {
+ $formWord = "$key{$this->sep}other";
+ $value = $form;
+ }
+
+ if ( !isset( $alts[$formWord] ) ) {
+ $alts[$formWord] = $message;
+ }
+
+ $string = $alts[$formWord];
+ $alts[$formWord] = str_replace( $ph, $value,
$string );
+ }
+ }
+
+ // Replace other variables.
+ foreach ( $alts as &$value ) {
+ $value = str_replace( array_keys( $placeholders ),
array_values( $placeholders ), $value );
+ }
+
+ if ( !isset( $alts["$key{$this->sep}other"] ) ) {
+ wfWarn( "Other not set for key $key" );
+ }
+
+ return $alts;
+ }
+
}
--
To view, visit https://gerrit.wikimedia.org/r/314679
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I15a9fb4a7f01f7ddb2f0b64c3a81062219dc2ac9
Gerrit-PatchSet: 4
Gerrit-Project: mediawiki/extensions/Translate
Gerrit-Branch: master
Gerrit-Owner: Eloquence <[email protected]>
Gerrit-Reviewer: Eloquence <[email protected]>
Gerrit-Reviewer: Nikerabbit <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
