Eloquence has uploaded a new change for review.
https://gerrit.wikimedia.org/r/314679
Change subject: Support for CLDR plural keywords in JSON file format
......................................................................
Support for CLDR plural keywords in JSON file format
This adds a configuration flag, parseCLDRPlurals, for the JSON
format. If set to yes, we will flatten/unflatten plural keywords
in the JSON source similarly to how it's already done for YAML,
using largely identical code (but also supporting the separator
option).
This makes redundant the YAML subclass implementation, which will
be refactored out in a follow-up change.
Change-Id: I15a9fb4a7f01f7ddb2f0b64c3a81062219dc2ac9
---
M ffs/JsonFFS.php
M utils/ArrayFlattener.php
2 files changed, 212 insertions(+), 16 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Translate
refs/changes/79/314679/1
diff --git a/ffs/JsonFFS.php b/ffs/JsonFFS.php
index 9e89d19..b94a7ba 100644
--- a/ffs/JsonFFS.php
+++ b/ffs/JsonFFS.php
@@ -49,8 +49,11 @@
unset( $messages['@metadata'] );
if ( isset( $this->extra['nestingSeparator'] ) ) {
- $flattener = new ArrayFlattener(
$this->extra['nestingSeparator'] );
- $messages = $flattener->flatten( $messages );
+ $parseCLDRPlurals = isset(
$this->extra['parseCLDRPlurals'] ) ?
+ $this->extra['parseCLDRPlurals'] : false;
+ $flattener = new ArrayFlattener(
$this->extra['nestingSeparator'],
+ $parseCLDRPlurals );
+ $messages = $flattener->flatten( $messages );
}
$messages = $this->group->getMangler()->mangle( $messages );
@@ -111,8 +114,11 @@
}
if ( isset( $this->extra['nestingSeparator'] ) ) {
- $flattener = new ArrayFlattener(
$this->extra['nestingSeparator'] );
- $messages = $flattener->unflatten( $messages );
+ $parseCLDRPlurals = isset(
$this->extra['parseCLDRPlurals'] ) ?
+ $this->extra['parseCLDRPlurals'] : false;
+ $flattener = new ArrayFlattener(
$this->extra['nestingSeparator'],
+ $parseCLDRPlurals );
+ $messages = $flattener->unflatten( $messages );
}
return FormatJson::encode( $messages, "\t", FormatJson::ALL_OK
) . "\n";
@@ -129,6 +135,9 @@
'nestingSeparator' =>
array(
'_type' =>
'text',
),
+ 'parseCLDRPlurals' =>
array(
+ '_type' =>
'boolean',
+ )
)
)
)
diff --git a/utils/ArrayFlattener.php b/utils/ArrayFlattener.php
index 8845ef1..38fa4fb 100644
--- a/utils/ArrayFlattener.php
+++ b/utils/ArrayFlattener.php
@@ -1,9 +1,11 @@
<?php
/**
- * Support for JSON message file format.
+ * Flattens message arrays for further processing. Supports parsing CLDR
+ * plural messages and converting them into MediaWiki's {{PLURAL}} syntax
+ * in a single message.
*
* @file
- * @author Niklas Laxström
+ * @author Niklas Laxström, Erik Moeller
* @license GPL-2.0+
* @since 2016.01
*/
@@ -11,14 +13,25 @@
class ArrayFlattener {
protected $sep;
- public function __construct( $sep = '.' ) {
+ // For CLDR pluralization rules
+ protected static $pluralWords = array(
+ 'zero' => 1,
+ 'one' => 1,
+ 'many' => 1,
+ 'few' => 1,
+ 'other' => 1,
+ 'two' => 1
+ );
+
+ public function __construct( $sep = '.', $parseCLDRPlurals = false ) {
$this->sep = $sep;
+ $this->parseCLDRPlurals = $parseCLDRPlurals;
}
/**
* Flattens multidimensional array.
*
- * @param array $unflat It's an array.
+ * @param array $unflat Array of messages
* @return array
*/
public function flatten( array $unflat ) {
@@ -30,14 +43,21 @@
continue;
}
- // Placeholder for special plural processing
-
- $temp = array();
- foreach ( $value as $subKey => $subValue ) {
- $newKey = "$key{$this->sep}$subKey";
- $temp[$newKey] = $subValue;
+ $plurals = false;
+ if ( $this->parseCLDRPlurals ) {
+ $plurals = $this->flattenCLDRPlurals( $value );
}
- $flat += $this->flatten( $temp );
+
+ if ( $this->parseCLDRPlurals && $plurals ) {
+ $flat[$key] = $plurals;
+ } else {
+ $temp = array();
+ foreach ( $value as $subKey => $subValue ) {
+ $newKey = "$key{$this->sep}$subKey";
+ $temp[$newKey] = $subValue;
+ }
+ $flat += $this->flatten( $temp );
+ }
// Can as well keep only one copy around.
unset( $unflat[$key] );
@@ -47,15 +67,87 @@
}
/**
+ * Flattens arrays that contain CLDR plural keywords into single values
using
+ * MediaWiki's plural syntax.
+ *
+ * @param array $messages Array of messages
+ *
+ * @throws MWException
+ * @return bool|string
+ */
+ public function flattenCLDRPlurals( $messages ) {
+
+ $pluralKeys = false;
+ $nonPluralKeys = false;
+ foreach ( $messages as $key => $value ) {
+ if ( is_array( $value ) ) {
+ // Plurals can only happen in the lowest level
of the structure
+ return false;
+ }
+
+ // Check if we find any reserved plural keyword
+ if ( isset( self::$pluralWords[$key] ) ) {
+ $pluralKeys = true;
+ } else {
+ $nonPluralKeys = true;
+ }
+ }
+
+ // No plural keys at all, we can skip
+ if ( !$pluralKeys ) {
+ return false;
+ }
+
+ // Mixed plural keys with other keys, should not happen
+ if ( $nonPluralKeys ) {
+ $keys = implode( ', ', array_keys( $messages ) );
+ throw new MWException( "Reserved plural keywords mixed
with other keys: $keys." );
+ }
+
+ $pls = '{{PLURAL';
+ foreach ( $messages as $key => $value ) {
+ if ( $key === 'other' ) {
+ continue;
+ }
+
+ $pls .= "|$key=$value";
+ }
+
+ // Put the "other" alternative last, without other= prefix.
+ $other = isset( $messages['other'] ) ? '|' . $messages['other']
: '';
+ $pls .= "$other}}";
+
+ return $pls;
+ }
+
+ /**
* Performs the reverse operation of flatten.
*
- * @param array $flat It's an array
+ * @param array $flat Array of messages
* @return array
*/
public function unflatten( $flat ) {
+
$unflat = array();
+ if ( $this->parseCLDRPlurals ) {
+ $unflattenedPlurals = array();
+ foreach ( $flat as $key => $value ) {
+ $plurals = false;
+ if ( !is_array( $value ) ) {
+ $plurals = $this->unflattenCLDRPlurals(
$key, $value );
+ }
+ if ( $plurals ) {
+ $unflattenedPlurals += $plurals;
+ } else {
+ $unflattenedPlurals[$key] = $value;
+ }
+ }
+ $flat = $unflattenedPlurals;
+ }
+
foreach ( $flat as $key => $value ) {
+
$path = explode( $this->sep, $key );
if ( count( $path ) === 1 ) {
$unflat[$key] = $value;
@@ -86,4 +178,99 @@
return $unflat;
}
+
+ /**
+ * Converts the MediaWiki plural syntax to array of CLDR style plurals
+ *
+ * @param string $key Message key prefix
+ * @param string $message The plural string
+ *
+ * @return bool|array
+ */
+ public function unflattenCLDRPlurals( $key, $message ) {
+
+ // Quick escape.
+ if ( strpos( $message, '{{PLURAL' ) === false ) {
+ return false;
+ }
+
+ /*
+ * Replace all variables with placeholders. Possible source of
bugs
+ * if other characters that given below are used.
+ */
+ $regex = '~\{[a-zA-Z_-]+}~';
+ $placeholders = array();
+ $match = array();
+
+ while ( preg_match( $regex, $message, $match ) ) {
+ $uniqkey = TranslateUtils::getPlaceholder();
+ $placeholders[$uniqkey] = $match[0];
+ $search = preg_quote( $match[0], '~' );
+ $message = preg_replace( "~$search~", $uniqkey,
$message );
+ }
+
+ // Then replace (possible multiple) plural instances into
placeholders.
+ $regex = '~\{\{PLURAL\|(.*?)}}~s';
+ $matches = array();
+ $match = array();
+
+ while ( preg_match( $regex, $message, $match ) ) {
+ $uniqkey = TranslateUtils::getPlaceholder();
+ $matches[$uniqkey] = $match;
+ $message = preg_replace( $regex, $uniqkey, $message, 1
);
+ }
+
+ // No plurals, should not happen.
+ if ( !count( $matches ) ) {
+ return false;
+ }
+
+ // The final array of alternative plurals forms.
+ $alts = array();
+
+ /*
+ * Then loop trough each plural block and replacing the
placeholders
+ * to construct the alternatives. Produces invalid output if
there is
+ * multiple plural bocks which don't have the same set of keys.
+ */
+ $pluralChoice = implode( '|', array_keys( self::$pluralWords )
);
+ $regex = "~($pluralChoice)\s*=\s*(.+)~s";
+ foreach ( $matches as $ph => $plu ) {
+ $forms = explode( '|', $plu[1] );
+
+ foreach ( $forms as $form ) {
+ if ( $form === '' ) {
+ continue;
+ }
+
+ $match = array();
+ if ( preg_match( $regex, $form, $match ) ) {
+ $formWord =
"$key{$this->sep}{$match[1]}";
+ $value = $match[2];
+ } else {
+ $formWord = "$key{$this->sep}other";
+ $value = $form;
+ }
+
+ if ( !isset( $alts[$formWord] ) ) {
+ $alts[$formWord] = $message;
+ }
+
+ $string = $alts[$formWord];
+ $alts[$formWord] = str_replace( $ph, $value,
$string );
+ }
+ }
+
+ // Replace other variables.
+ foreach ( $alts as &$value ) {
+ $value = str_replace( array_keys( $placeholders ),
array_values( $placeholders ), $value );
+ }
+
+ if ( !isset( $alts["$key{$this->sep}other"] ) ) {
+ wfWarn( "Other not set for key $key" );
+ }
+
+ return $alts;
+ }
+
}
--
To view, visit https://gerrit.wikimedia.org/r/314679
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I15a9fb4a7f01f7ddb2f0b64c3a81062219dc2ac9
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Translate
Gerrit-Branch: master
Gerrit-Owner: Eloquence <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
