Yurik has uploaded a new change for review. https://gerrit.wikimedia.org/r/291159
Change subject: Implemented JCTitleValue with parsing ...................................................................... Implemented JCTitleValue with parsing * JCTitleValue inherits from TitleValue, but also stores the JC configuration matching this title. * Added JCSingleton::parseTitle($value,$ns) This method tries to convert the value from various forms into JCTitleValue. In case $value is a string, it parses it using a custom MediaWikiTitleCodec instance, which it creates as close to "invariant" as possible. The string is prepended with the value from the setting $wgJsonConfigInterwikiPrefix, which should be set to a valid non-local interwiki prefix. * Unit tests to check this whole thing. Change-Id: I5047e60141f4fa17cee5e43b5e96c2538aa19837 --- M JsonConfig.php M includes/JCApi.php M includes/JCCache.php M includes/JCSingleton.php A includes/JCTitleValue.php M tests/phpunit/JCLoaderTest.php A tests/phpunit/JCTitleValueParsingTest.php 7 files changed, 302 insertions(+), 89 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/JsonConfig refs/changes/59/291159/1 diff --git a/JsonConfig.php b/JsonConfig.php index 2ae8369..c43da16 100644 --- a/JsonConfig.php +++ b/JsonConfig.php @@ -21,7 +21,7 @@ $wgExtensionCredits['other'][] = array( 'path' => __FILE__, 'name' => 'JsonConfig', - 'version' => '1.0.0', + 'version' => '1.1.0', 'author' => array( 'Yuri Astrakhan' ), 'descriptionmsg' => 'jsonconfig-desc', 'url' => 'https://www.mediawiki.org/wiki/Extension:JsonConfig', @@ -53,6 +53,7 @@ 'JCSingleton', 'JCTabularContent', 'JCTabularContentView', + 'JCTitleValue', 'JCUtils', 'JCValidators', 'JCValue', @@ -61,6 +62,15 @@ } /** + * Use this prefix when parsing remote titles. This prefix will be prepended to title strings + * in JCSingleton::parseTitle(), passed through the english TitleParser, and prefix removed. + * "commons:" seems to be a fairly safe bet, as it is auto-setup in Vagrant and in many wikis. + * Make sure this prefix exists in the interwiki table: + * http://.../w/api.php?action=query&meta=siteinfo&siprop=interwikimap + */ +$wgJsonConfigInterwikiPrefix = 'commons'; + +/** * Each extension should add its configuration profiles as described in the doc * https://www.mediawiki.org/wiki/Requests_for_comment/Json_Config_pages_in_wiki * https://www.mediawiki.org/wiki/Extension:JsonConfig diff --git a/includes/JCApi.php b/includes/JCApi.php index 1cbb44f..818723b 100644 --- a/includes/JCApi.php +++ b/includes/JCApi.php @@ -99,17 +99,15 @@ // extra whitespaces at either end. if ( $ns === null || !array_key_exists( $ns, JCSingleton::getTitleMap() ) || ( $t = \Title::newFromText( $parts[1], $ns ) ) === null || - ( $titleValue = $t->getTitleValue() ) === null || - !( $conf = JCSingleton::getMetadata( $titleValue ) ) + !( $jtv = JCSingleton::parseTitle( $t ) ) ) { $this->dieUsage( 'The "title" parameter must be in form NS:Title, where NS is either an integer or a canonical ' . 'namespace name. In either case, namespace must be defined as part of JsonConfig configuration', 'badparam-titles' ); } - /** @var \stdClass $conf */ - /** @var \TitleValue $titleValue */ - $handler = new JCContentHandler( $conf->model ); + /** @var JCTitleValue $jtv */ + $handler = new JCContentHandler( $jtv->getConfig()->model ); if ( isset( $params['content'] ) && $params['content'] !== '' ) { if ( $command !== 'reload ' ) { @@ -122,7 +120,7 @@ $content = false; } - $jc = new JCCache( $titleValue, $conf, $content ); + $jc = new JCCache( $jtv, $content ); if ( $command === 'reset' ) { $jc->resetCache( false ); // clear cache } elseif ( $content ) { diff --git a/includes/JCCache.php b/includes/JCCache.php index 551b3e2..c1bcdb4 100644 --- a/includes/JCCache.php +++ b/includes/JCCache.php @@ -2,8 +2,6 @@ namespace JsonConfig; use MWNamespace; -use stdClass; -use TitleValue; /** * Represents a json blob on a remote wiki. @@ -11,8 +9,6 @@ */ class JCCache { private $titleValue, $key, $cache; - /** @var stdClass */ - private $conf; /** @var bool|string|JCContent */ private $content = null; @@ -24,14 +20,13 @@ * Constructor for JCCache * ** DO NOT USE directly - call JCSingleton::getCachedStore() instead. ** * - * @param TitleValue $titleValue - * @param stdClass $conf + * @param JCTitleValue $titleValue * @param JCContent $content */ - function __construct( $titleValue, $conf, $content = null ) { + function __construct( JCTitleValue $titleValue, $content = null ) { global $wgJsonConfigCacheKeyPrefix; $this->titleValue = $titleValue; - $this->conf = $conf; + $conf = $this->titleValue->getConfig(); $flRev = $conf->flaggedRevs; $key = implode( ':', array( 'JsonConfig', @@ -57,7 +52,7 @@ if ( $this->content === null ) { $value = $this->memcGet(); // Get content from the memcached if ( $value === false ) { - if ( $this->conf->store ) { + if ( $this->titleValue->getConfig()->store ) { $this->loadLocal(); // Get it from the local wiki } else { $this->loadRemote(); // Get it from HTTP @@ -114,9 +109,10 @@ public function resetCache( $updateCacheContent = null ) { global $wgJsonConfigDisableCache; if ( !$wgJsonConfigDisableCache ) { + $conf = $this->titleValue->getConfig(); if ( $this->content && ( $updateCacheContent === true || - ( $updateCacheContent === null && isset( $this->conf->store ) && - $this->conf->store->cacheNewValue ) ) + ( $updateCacheContent === null && isset( $conf->store ) && + $conf->store->cacheNewValue ) ) ) { $this->memcSet(); // update cache with the new value } else { @@ -152,16 +148,17 @@ private function loadRemote() { do { $result = false; - $remote = $this->conf->remote; + $conf = $this->titleValue->getConfig(); + $remote = $conf->remote; $req = JCUtils::initApiRequestObj( $remote->url, $remote->username, $remote->password ); if ( !$req ) { break; } $ns = - $this->conf->nsName ? $this->conf->nsName + $conf->nsName ? $conf->nsName : MWNamespace::getCanonicalName( $this->titleValue->getNamespace() ); $articleName = $ns . ':' . $this->titleValue->getText(); - $flrevs = $this->conf->flaggedRevs; + $flrevs = $conf->flaggedRevs; // if flaggedRevs is false, get wiki page directly, otherwise get the flagged state first $res = $this->getPageFromApi( $articleName, $req, $flrevs === false ? array( diff --git a/includes/JCSingleton.php b/includes/JCSingleton.php index 74f6906..820aef0 100644 --- a/includes/JCSingleton.php +++ b/includes/JCSingleton.php @@ -1,9 +1,18 @@ <?php namespace JsonConfig; +use Article; use ContentHandler; use Exception; +use GenderCache; +use Language; +use MalformedTitleException; use MapCacheLRU; +use MediaWiki\Linker\LinkTarget; +use MediaWikiTitleCodec; +use MWException; +use TitleParser; +use Status; use stdClass; use TitleValue; use Title; @@ -34,9 +43,19 @@ static $namespaces = array(); /** - * @var MapCacheLRU[] contains a cache of recently requested content objects as namespace => cache + * @var MapCacheLRU[] contains a cache of recently resolved JCTitleValues as namespace => MapCacheLRU + */ + static $titleMapCacheLru = array(); + + /** + * @var MapCacheLRU[] contains a cache of recently requested content objects as namespace => MapCacheLRU */ static $mapCacheLru = array(); + + /** + * @var TitleParser cached invariant title parser + */ + static $titleParser; /** * Initializes singleton state by parsing $wgJsonConfig* values @@ -305,7 +324,7 @@ * Get content object for the given title. * Namespace ID does not need to be defined in the current wiki, * as long as it is defined in $wgJsonConfigs. - * @param TitleValue $titleValue + * @param TitleValue|JCTitleValue $titleValue * @return bool|JCContent Returns false if the title is not handled by the settings */ public static function getContent( TitleValue $titleValue ) { @@ -313,13 +332,13 @@ $content = self::getContentFromLocalCache( $titleValue ); if ( $content === null ) { - $conf = self::getMetadata( $titleValue ); - if ( $conf ) { - $store = new JCCache( $titleValue, $conf ); + $jtv = self::parseTitle( $titleValue ); + if ( $jtv ) { + $store = new JCCache( $jtv ); $content = $store->get(); if ( is_string( $content ) ) { // Convert string to the content object if needed - $handler = new JCContentHandler( $conf->model ); + $handler = new JCContentHandler( $jtv->getConfig()->model ); $content = $handler->unserializeContent( $content, null, false ); } } else { @@ -341,9 +360,9 @@ */ public static function parseContent( TitleValue $titleValue, $jsonText ) { - $conf = self::getMetadata( $titleValue ); - if ( $conf ) { - $handler = new JCContentHandler( $conf->model ); + $jtv = self::parseTitle( $titleValue ); + if ( $jtv ) { + $handler = new JCContentHandler( $jtv->getConfig()->model ); return $handler->unserializeContent( $jsonText, null, false ); } @@ -352,7 +371,7 @@ /** * Mostly for debugging purposes, this function returns initialized internal JsonConfig settings - * @return array + * @return array[] map of namespaceIDs to list of configurations */ public static function getTitleMap() { self::init(); @@ -381,38 +400,120 @@ } /** + * Given a title (either a user-given string, or as an object), return JCTitleValue + * @param Title|TitleValue|string|Article $value + * @param int|null $namespace Only used when title is a string + * @return JCTitleValue|null|false false if unrecognized namespace, + * and null if namespace is handled but does not match this title + * @throws MWException + */ + public static function parseTitle( $value, $namespace = null ) { + if ( $value === null || $value === '' || $value === false ) { + // In some weird cases $value is null + return false; + } elseif ( $value instanceof JCTitleValue ) { + // Nothing to do + return $value; + } elseif ( $namespace !== null && !is_integer( $namespace ) ) { + throw new MWException( '$namespace parameter must be either null or an integer' ); + } + + // figure out the namespace ID (int) - we don't need to parse the string if ns is unknown + if ( $value instanceof LinkTarget ) { + if ( $namespace === null ) { + $namespace = $value->getNamespace(); + } + } elseif ( is_string( $value ) ) { + if ( $namespace === null ) { + throw new MWException( '$namespace parameter is missing for string $value' ); + } + } elseif ( method_exists( $value, 'getTitle' ) ) { + $value = $value->getTitle(); + if ( $namespace === null ) { + $namespace = $value->getNamespace(); + } + } else { + wfLogWarning( 'Unexpected title param type ' . gettype( $value ) ); + return false; + } + + // Search title map for the matching configuration + $map = self::getTitleMap(); + if ( array_key_exists( $namespace, $map ) ) { + + // Get appropriate LRU cache object + if ( !array_key_exists( $namespace, self::$titleMapCacheLru ) ) { + self::$titleMapCacheLru[$namespace] = $cache = new MapCacheLRU( 20 ); + } else { + $cache = self::$titleMapCacheLru[$namespace]; + } + + // Parse string if needed + // TODO: should the string parsing also be cached? + if ( is_string( $value ) ) { + if ( !self::$titleParser ) { + self::$titleParser = + new MediaWikiTitleCodec( Language::factory( 'en' ), new GenderCache() ); + } + // Major hack, but until MediaWikiTitleCodec has global state, I can't think of a + // better way. Interwiki prefixes are a special case for title parsing: + // first letter is not capitalized, namespaces are not resolved, etc. + // So we prepend an interwiki prefix to fool title codec, and later remove it. + global $wgJsonConfigInterwikiPrefix; + try { + $value = $wgJsonConfigInterwikiPrefix . ':' . $value; + $parts = self::$titleParser->splitTitleString( $value ); + if ( $parts['dbkey'] === '' || $parts['namespace'] !== 0 || + $parts['fragment'] !== '' || $parts['local_interwiki'] !== false || + $parts['interwiki'] !== $wgJsonConfigInterwikiPrefix + ) { + return null; + } + } catch ( MalformedTitleException $e ) { + return null; + } + $dbKey = $parts['dbkey']; + } else { + $dbKey = $value->getDBkey(); + } + + // A bit weird here: cache will store JCTitleValue objects or false if the namespace + // is known to JsonConfig but the dbkey does not match. But in case the title is not + // handled, this function returns null instead of false if the namespace is known, and false otherwise + $result = $cache->get( $dbKey ); + if ( $result === null ) { + + $result = false; + foreach ( $map[$namespace] as $conf ) { + $re = $conf->pattern; + if ( !$re || preg_match( $re, $dbKey ) ) { + $result = new JCTitleValue( $namespace, $dbKey, $conf ); + break; + } + } + + $cache->set( $dbKey, $result ); + } + + // return null if the given namespace is mentioned in the config, but title doesn't match + return $result ?: null; + + } else { + // return false if JC doesn't know anything about this namespace + return false; + } + } + + /** * Returns an array with settings if the $titleValue object is handled by the JsonConfig extension, * false if unrecognized namespace, and null if namespace is handled but not this title * @param TitleValue $titleValue * @return stdClass|false|null + * @obsolete use JCSingleton::parseTitle() instead */ public static function getMetadata( $titleValue ) { - static $lastTitle = null; - static $lastResult = false; - if ( !$titleValue ) { - return false; // It is possible to have a null TitleValue (bug 66555) - } elseif ( $titleValue === $lastTitle ) { - return $lastResult; - } - $lastTitle = $titleValue; - $ns = $titleValue->getNamespace(); - /** @var array[] $map array of: { namespace => [ configs ] } */ - $map = self::getTitleMap(); - if ( array_key_exists( $ns, $map ) ) { - $text = $titleValue->getText(); - foreach ( $map[$ns] as $conf ) { - $re = $conf->pattern; - if ( !$re || preg_match( $re, $text ) ) { - $lastResult = $conf; - return $lastResult; - } - } - // We know about the namespace, but there is no specific configuration - $lastResult = null; - return $lastResult; - } - $lastResult = false; - return $lastResult; + $tv = JCSingleton::parseTitle( $titleValue ); + return $tv ? $tv->getConfig() : $tv; } /** @@ -449,9 +550,9 @@ * @return bool */ public static function onContentHandlerDefaultModelFor( $title, &$modelId ) { - $conf = self::getMetadata( $title->getTitleValue() ); - if ( $conf ) { - $modelId = $conf->model; + $jtv = self::parseTitle( $title ); + if ( $jtv ) { + $modelId = $jtv->getConfig()->model; return false; } return true; @@ -483,7 +584,7 @@ */ static function onCodeEditorGetPageLanguage( $title, &$lang ) { $handler = ContentHandler::getForModelID( $title->getContentModel() ); - if ( $handler->getDefaultFormat() === CONTENT_FORMAT_JSON || self::getMetadata( $title->getTitleValue() ) ) { + if ( $handler->getDefaultFormat() === CONTENT_FORMAT_JSON || self::parseTitle( $title ) ) { $lang = 'json'; } return true; @@ -505,7 +606,7 @@ if ( is_a( $content, 'JsonConfig\JCContent' ) ) { $status->merge( $content->getStatus() ); if ( !$status->isGood() ) { - $status->ok = false; + $status->setResult( false, $status->getValue() ); } } return true; @@ -521,23 +622,23 @@ $title = $out->getTitle(); $handler = ContentHandler::getForModelID( $title->getContentModel() ); if ( $handler->getDefaultFormat() === CONTENT_FORMAT_JSON || - self::getMetadata( $title->getTitleValue() ) + self::parseTitle( $title ) ) { $out->addModuleStyles( 'ext.jsonConfig' ); } return true; } - public static function onMovePageIsValidMove( Title $oldTitle, Title $newTitle, \Status $status ) { - $conf = self::getMetadata( $oldTitle->getTitleValue() ); - if ( $conf ) { - $newConf = self::getMetadata( $newTitle->getTitleValue() ); - if ( !$newConf ) { - // @todo: is parse() the right func to use here? + public static function onMovePageIsValidMove( Title $oldTitle, Title $newTitle, Status $status ) { + $jtv = self::parseTitle( $oldTitle ); + if ( $jtv ) { + $newJtv = self::parseTitle( $newTitle ); + if ( !$newJtv ) { $status->fatal( 'jsonconfig-move-aborted-ns' ); return false; - } elseif ( $conf->model !== $newConf->model ) { - $status->fatal( 'jsonconfig-move-aborted-model', $conf->model, $newConf->model ); + } elseif ( $jtv->getConfig()->model !== $newJtv->getConfig()->model ) { + $status->fatal( 'jsonconfig-move-aborted-model', $jtv->getConfig()->model, + $newJtv->getConfig()->model ); return false; } } @@ -584,7 +685,7 @@ * @return bool */ public static function onuserCan( /** @noinspection PhpUnusedParameterInspection */ &$title, &$user, $action, &$result = null ) { - if ( $action === 'create' && self::getMetadata( $title->getTitleValue() ) === null ) { + if ( $action === 'create' && self::parseTitle( $title ) === null ) { // prohibit creation of the pages for the namespace that we handle, // if the title is not matching declared rules $result = false; @@ -600,24 +701,15 @@ */ private static function onArticleChangeComplete( $value, $content = null ) { if ( $value && ( !$content || is_a( $content, 'JsonConfig\JCContent' ) ) ) { - /** @var TitleValue $tv */ - if ( method_exists( $value, 'getTitleValue') ) { - $tv = $value->getTitleValue(); - } elseif ( method_exists( $value, 'getTitle')) { - $tv = $value->getTitle()->getTitleValue(); - } else { - wfLogWarning( 'Unknown object type ' . gettype( $value ) ); - return true; - } - $conf = self::getMetadata( $tv ); - if ( $conf && $conf->store ) { - $store = new JCCache( $tv, $conf, $content ); + $jtv = self::parseTitle( $value ); + if ( $jtv && $jtv->getConfig()->store ) { + $store = new JCCache( $jtv, $content ); $store->resetCache(); // Handle remote site notification - if ( $conf->store->notifyUrl ) { - $store = $conf->store; + $store = $jtv->getConfig()->store; + if ( $store->notifyUrl ) { $req = JCUtils::initApiRequestObj( $store->notifyUrl, $store->notifyUsername, $store->notifyPassword ); @@ -626,7 +718,7 @@ 'format' => 'json', 'action' => 'jsonconfig', 'command' => 'reload', - 'title' => $tv->getNamespace() . ':' . $tv->getDBkey(), + 'title' => $jtv->getNamespace() . ':' . $jtv->getDBkey(), ); JCUtils::callApi( $req, $query, 'notify remote JsonConfig client' ); } diff --git a/includes/JCTitleValue.php b/includes/JCTitleValue.php new file mode 100644 index 0000000..6e777bc --- /dev/null +++ b/includes/JCTitleValue.php @@ -0,0 +1,37 @@ +<?php +namespace JsonConfig; + +use MWException; +use stdClass; +use TitleValue; + +/** + * A value object class that contains namespace ID, title, and the corresponding jsonconfig configuration + * @package JsonConfig + */ +final class JCTitleValue extends TitleValue { + + /** + * @var stdClass + */ + private $config; + + /** + * JCTitleValue constructor. + * @param int $namespace Possibly belonging to a foreign wiki + * @param string $dbkey + * @param stdClass $config JsonConfig configuration object + * @throws MWException + */ + public function __construct( $namespace, $dbkey, $config ) { + if ( $namespace !== $config->namespace ) { + throw new MWException( 'Namespace does not match config' ); + } + parent::__construct( $namespace, $dbkey ); + $this->config = $config; + } + + public function getConfig() { + return $this->config; + } +} diff --git a/tests/phpunit/JCLoaderTest.php b/tests/phpunit/JCLoaderTest.php index 688a02c..29f1d38 100644 --- a/tests/phpunit/JCLoaderTest.php +++ b/tests/phpunit/JCLoaderTest.php @@ -33,8 +33,8 @@ [], self::getExpectedObj( [], true, false ) ], [ - [ 'pattern' => '^.+\.json$' ], - self::getExpectedObj( [ 'pattern' => '^.+\.json$' ], true, false ) + [ 'pattern' => '\.json$' ], + self::getExpectedObj( [ 'pattern' => '/\.json$/' ], true, false ) ], [ [ 'namespace' => 400, 'nsName' => 'abc' ], diff --git a/tests/phpunit/JCTitleValueParsingTest.php b/tests/phpunit/JCTitleValueParsingTest.php new file mode 100644 index 0000000..529bb6e --- /dev/null +++ b/tests/phpunit/JCTitleValueParsingTest.php @@ -0,0 +1,79 @@ +<?php +namespace JsonConfig\Tests; + +use JsonConfig\JCSingleton; +use MediaWikiTestCase; + +/** + * @package JsonConfigTests + * @group JsonConfig +*/ +class JCTitleValueParsingTest extends MediaWikiTestCase { + + private static $configBackup; + + public static function setUpBeforeClass() { + JCSingleton::getTitleMap(); // Initialize internal Init() flag + self::$configBackup = [ JCSingleton::$titleMap, JCSingleton::$namespaces ]; + + list( JCSingleton::$titleMap, JCSingleton::$namespaces ) = + JCSingleton::parseConfiguration( [ 'modelForNs0', 'modelForNs1' ], + [ 'globalModel' => 'something' ], [ + 'model1' => [ 'nsName' => 'All', 'namespace' => 800 ], + 'model2' => [ + 'nsName' => 'Dat', + 'namespace' => 900, + 'pattern' => '/^(lower|Capitalized|sub\/space|with:colon)$/' + ], + ], [ + 'model1' => null, + 'model2' => null, + 'globalModel' => 'conflicts with JsonConfig models', + ] ); + } + + public static function tearDownAfterClass() { + list( JCSingleton::$titleMap, JCSingleton::$namespaces ) = self::$configBackup; + } + + /** @dataProvider provideValues + * @param $value + * @param $ns + * @param bool|array $expected + */ + public function testTitleParsing( $value, $ns, $expected = false ) { + + $actual = JCSingleton::parseTitle( $value, $ns ); + if ( !$expected ) { + $this->assertSame( $expected, $actual ); + } else { + $this->assertInstanceOf( 'JsonConfig\JCTitleValue', $actual ); + $this->assertSame( $expected, $actual->getDBkey() ); + $this->assertSame( $ns, $actual->getNamespace() ); + $this->assertNotNull( $actual->getConfig() ); + } + } + + public function provideValues() { + return [ + // title, ns, expected + // 900: lower|Capitalized|sub/space|with:colon + [ false, null, false ], + [ null, null, false ], + [ '', null, false ], + [ '_', 0, false ], + [ '_', 800, null ], + [ '_', 900, null ], + [ ':a/b\d e_a ', 800, 'a/b\d_e_a' ], // normalization + [ 'wikipedia:ok', 800, 'wikipedia:ok' ], // known interwiki + [ 'nope', 900, null ], + [ 'Lower', 900, null ], + [ 'lower', 900, 'lower' ], + [ 'capitalized', 900, null ], + [ 'Capitalized', 900, 'Capitalized' ], + [ 'sub/space', 900, 'sub/space' ], + [ 'with:colon', 900, 'with:colon' ], + ]; + } + +} -- To view, visit https://gerrit.wikimedia.org/r/291159 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I5047e60141f4fa17cee5e43b5e96c2538aa19837 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/extensions/JsonConfig Gerrit-Branch: master Gerrit-Owner: Yurik <yu...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits