EBernhardson has uploaded a new change for review. https://gerrit.wikimedia.org/r/244077
Change subject: Revert "Revert "Update CirrusSearch config for testwiki to talk to second cluster"" ...................................................................... Revert "Revert "Update CirrusSearch config for testwiki to talk to second cluster"" This reverts commit eca11c19e47bde446cafa6ff792a634fe1667eb9. Change-Id: I3566d40bf93f5766d0825bb63f9804550ae19d33 --- A tests/Defines.php A tests/SiteConfiguration.php M tests/bootstrap.php A tests/cirrusTest.php M tests/dbconfigTest.php M wmf-config/CirrusSearch-common.php M wmf-config/CirrusSearch-labs.php M wmf-config/CirrusSearch-production.php M wmf-config/CommonSettings.php M wmf-config/InitialiseSettings-labs.php M wmf-config/InitialiseSettings.php 11 files changed, 1,040 insertions(+), 16 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/mediawiki-config refs/changes/77/244077/1 diff --git a/tests/Defines.php b/tests/Defines.php new file mode 100644 index 0000000..c24b3c2 --- /dev/null +++ b/tests/Defines.php @@ -0,0 +1,305 @@ +<?php +/** + * A few constants that might be needed during LocalSettings.php. + * + * This lives in mediawiki but was copied into the mediawiki-config + * repository to allow tests to run in isolation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * @defgroup Constants MediaWiki constants + */ + +/**@{ + * Database related constants + */ +define( 'DBO_DEBUG', 1 ); +define( 'DBO_NOBUFFER', 2 ); +define( 'DBO_IGNORE', 4 ); +define( 'DBO_TRX', 8 ); // automatically start transaction on first query +define( 'DBO_DEFAULT', 16 ); +define( 'DBO_PERSISTENT', 32 ); +define( 'DBO_SYSDBA', 64 ); // for oracle maintenance +define( 'DBO_DDLMODE', 128 ); // when using schema files: mostly for Oracle +define( 'DBO_SSL', 256 ); +define( 'DBO_COMPRESS', 512 ); +/**@}*/ + +/**@{ + * Valid database indexes + * Operation-based indexes + */ +define( 'DB_SLAVE', -1 ); # Read from the slave (or only server) +define( 'DB_MASTER', -2 ); # Write to master (or only server) +/**@}*/ + +# Obsolete aliases +define( 'DB_READ', -1 ); +define( 'DB_WRITE', -2 ); + +/**@{ + * Virtual namespaces; don't appear in the page database + */ +define( 'NS_MEDIA', -2 ); +define( 'NS_SPECIAL', -1 ); +/**@}*/ + +/**@{ + * Real namespaces + * + * Number 100 and beyond are reserved for custom namespaces; + * DO NOT assign standard namespaces at 100 or beyond. + * DO NOT Change integer values as they are most probably hardcoded everywhere + * see bug #696 which talked about that. + */ +define( 'NS_MAIN', 0 ); +define( 'NS_TALK', 1 ); +define( 'NS_USER', 2 ); +define( 'NS_USER_TALK', 3 ); +define( 'NS_PROJECT', 4 ); +define( 'NS_PROJECT_TALK', 5 ); +define( 'NS_FILE', 6 ); +define( 'NS_FILE_TALK', 7 ); +define( 'NS_MEDIAWIKI', 8 ); +define( 'NS_MEDIAWIKI_TALK', 9 ); +define( 'NS_TEMPLATE', 10 ); +define( 'NS_TEMPLATE_TALK', 11 ); +define( 'NS_HELP', 12 ); +define( 'NS_HELP_TALK', 13 ); +define( 'NS_CATEGORY', 14 ); +define( 'NS_CATEGORY_TALK', 15 ); + +/** + * NS_IMAGE and NS_IMAGE_TALK are the pre-v1.14 names for NS_FILE and + * NS_FILE_TALK respectively, and are kept for compatibility. + * + * When writing code that should be compatible with older MediaWiki + * versions, either stick to the old names or define the new constants + * yourself, if they're not defined already. + */ +define( 'NS_IMAGE', NS_FILE ); +define( 'NS_IMAGE_TALK', NS_FILE_TALK ); +/**@}*/ + +/**@{ + * Cache type + */ +define( 'CACHE_ANYTHING', -1 ); // Use anything, as long as it works +define( 'CACHE_NONE', 0 ); // Do not cache +define( 'CACHE_DB', 1 ); // Store cache objects in the DB +define( 'CACHE_MEMCACHED', 2 ); // MemCached, must specify servers in $wgMemCacheServers +define( 'CACHE_ACCEL', 3 ); // APC, XCache or WinCache +/**@}*/ + +/**@{ + * Media types. + * This defines constants for the value returned by File::getMediaType() + */ +// unknown format +define( 'MEDIATYPE_UNKNOWN', 'UNKNOWN' ); +// some bitmap image or image source (like psd, etc). Can't scale up. +define( 'MEDIATYPE_BITMAP', 'BITMAP' ); +// some vector drawing (SVG, WMF, PS, ...) or image source (oo-draw, etc). Can scale up. +define( 'MEDIATYPE_DRAWING', 'DRAWING' ); +// simple audio file (ogg, mp3, wav, midi, whatever) +define( 'MEDIATYPE_AUDIO', 'AUDIO' ); +// simple video file (ogg, mpg, etc; +// no not include formats here that may contain executable sections or scripts!) +define( 'MEDIATYPE_VIDEO', 'VIDEO' ); +// Scriptable Multimedia (flash, advanced video container formats, etc) +define( 'MEDIATYPE_MULTIMEDIA', 'MULTIMEDIA' ); +// Office Documents, Spreadsheets (office formats possibly containing apples, scripts, etc) +define( 'MEDIATYPE_OFFICE', 'OFFICE' ); +// Plain text (possibly containing program code or scripts) +define( 'MEDIATYPE_TEXT', 'TEXT' ); +// binary executable +define( 'MEDIATYPE_EXECUTABLE', 'EXECUTABLE' ); +// archive file (zip, tar, etc) +define( 'MEDIATYPE_ARCHIVE', 'ARCHIVE' ); +/**@}*/ + +/**@{ + * Antivirus result codes, for use in $wgAntivirusSetup. + */ +define( 'AV_NO_VIRUS', 0 ); # scan ok, no virus found +define( 'AV_VIRUS_FOUND', 1 ); # virus found! +define( 'AV_SCAN_ABORTED', -1 ); # scan aborted, the file is probably immune +define( 'AV_SCAN_FAILED', false ); # scan failed (scanner not found or error in scanner) +/**@}*/ + +/**@{ + * Anti-lock flags + * Was used by $wgAntiLockFlags, which was removed with 1.25 + * Constants kept to not have warnings when used in LocalSettings + */ +define( 'ALF_PRELOAD_LINKS', 1 ); // unused +define( 'ALF_PRELOAD_EXISTENCE', 2 ); // unused +define( 'ALF_NO_LINK_LOCK', 4 ); // unused +define( 'ALF_NO_BLOCK_LOCK', 8 ); // unused +/**@}*/ + +/**@{ + * Date format selectors; used in user preference storage and by + * Language::date() and co. + */ +define( 'MW_DATE_DEFAULT', 'default' ); +define( 'MW_DATE_MDY', 'mdy' ); +define( 'MW_DATE_DMY', 'dmy' ); +define( 'MW_DATE_YMD', 'ymd' ); +define( 'MW_DATE_ISO', 'ISO 8601' ); +/**@}*/ + +/**@{ + * RecentChange type identifiers + */ +define( 'RC_EDIT', 0 ); +define( 'RC_NEW', 1 ); +define( 'RC_LOG', 3 ); +define( 'RC_EXTERNAL', 5 ); +define( 'RC_CATEGORIZE', 6 ); +/**@}*/ + +/**@{ + * Article edit flags + */ +define( 'EDIT_NEW', 1 ); +define( 'EDIT_UPDATE', 2 ); +define( 'EDIT_MINOR', 4 ); +define( 'EDIT_SUPPRESS_RC', 8 ); +define( 'EDIT_FORCE_BOT', 16 ); +define( 'EDIT_DEFER_UPDATES', 32 ); +define( 'EDIT_AUTOSUMMARY', 64 ); +/**@}*/ + +/**@{ + * Flags for Database::makeList() + * These are also available as Database class constants + */ +define( 'LIST_COMMA', 0 ); +define( 'LIST_AND', 1 ); +define( 'LIST_SET', 2 ); +define( 'LIST_NAMES', 3 ); +define( 'LIST_OR', 4 ); +/**@}*/ + +/**@{ + * Hook support constants + */ +define( 'MW_SUPPORTS_PARSERFIRSTCALLINIT', 1 ); +define( 'MW_SUPPORTS_LOCALISATIONCACHE', 1 ); +define( 'MW_SUPPORTS_CONTENTHANDLER', 1 ); +define( 'MW_EDITFILTERMERGED_SUPPORTS_API', 1 ); +/**@}*/ + +/** Support for $wgResourceModules */ +define( 'MW_SUPPORTS_RESOURCE_MODULES', 1 ); + +/**@{ + * Allowed values for Parser::$mOutputType + * Parameter to Parser::startExternalParse(). + * Use of Parser consts is preferred: + * - Parser::OT_HTML + * - Parser::OT_WIKI + * - Parser::OT_PREPROCESS + * - Parser::OT_MSG + * - Parser::OT_PLAIN + */ +define( 'OT_HTML', 1 ); +define( 'OT_WIKI', 2 ); +define( 'OT_PREPROCESS', 3 ); +define( 'OT_MSG', 3 ); // b/c alias for OT_PREPROCESS +define( 'OT_PLAIN', 4 ); +/**@}*/ + +/**@{ + * Flags for Parser::setFunctionHook + * Use of Parser consts is preferred: + * - Parser::SFH_NO_HASH + * - Parser::SFH_OBJECT_ARGS + */ +define( 'SFH_NO_HASH', 1 ); +define( 'SFH_OBJECT_ARGS', 2 ); +/**@}*/ + +/**@{ + * Autopromote conditions (must be here and not in Autopromote.php, so that + * they're loaded for DefaultSettings.php before AutoLoader.php) + */ +define( 'APCOND_EDITCOUNT', 1 ); +define( 'APCOND_AGE', 2 ); +define( 'APCOND_EMAILCONFIRMED', 3 ); +define( 'APCOND_INGROUPS', 4 ); +define( 'APCOND_ISIP', 5 ); +define( 'APCOND_IPINRANGE', 6 ); +define( 'APCOND_AGE_FROM_EDIT', 7 ); +define( 'APCOND_BLOCKED', 8 ); +define( 'APCOND_ISBOT', 9 ); +/**@}*/ + +/** @{ + * Protocol constants for wfExpandUrl() + */ +define( 'PROTO_HTTP', 'http://' ); +define( 'PROTO_HTTPS', 'https://' ); +define( 'PROTO_RELATIVE', '//' ); +define( 'PROTO_CURRENT', null ); +define( 'PROTO_CANONICAL', 1 ); +define( 'PROTO_INTERNAL', 2 ); +/**@}*/ + +/**@{ + * Content model ids, used by Content and ContentHandler. + * These IDs will be exposed in the API and XML dumps. + * + * Extensions that define their own content model IDs should take + * care to avoid conflicts. Using the extension name as a prefix is recommended, + * for example 'myextension-somecontent'. + */ +define( 'CONTENT_MODEL_WIKITEXT', 'wikitext' ); +define( 'CONTENT_MODEL_JAVASCRIPT', 'javascript' ); +define( 'CONTENT_MODEL_CSS', 'css' ); +define( 'CONTENT_MODEL_TEXT', 'text' ); +define( 'CONTENT_MODEL_JSON', 'json' ); +/**@}*/ + +/**@{ + * Content formats, used by Content and ContentHandler. + * These should be MIME types, and will be exposed in the API and XML dumps. + * + * Extensions are free to use the below formats, or define their own. + * It is recommended to stick with the conventions for MIME types. + */ +// wikitext +define( 'CONTENT_FORMAT_WIKITEXT', 'text/x-wiki' ); +// for js pages +define( 'CONTENT_FORMAT_JAVASCRIPT', 'text/javascript' ); +// for css pages +define( 'CONTENT_FORMAT_CSS', 'text/css' ); +// for future use, e.g. with some plain-html messages. +define( 'CONTENT_FORMAT_TEXT', 'text/plain' ); +// for future use, e.g. with some plain-html messages. +define( 'CONTENT_FORMAT_HTML', 'text/html' ); +// for future use with the api and for extensions +define( 'CONTENT_FORMAT_SERIALIZED', 'application/vnd.php.serialized' ); +// for future use with the api, and for use by extensions +define( 'CONTENT_FORMAT_JSON', 'application/json' ); +// for future use with the api, and for use by extensions +define( 'CONTENT_FORMAT_XML', 'application/xml' ); +/**@}*/ diff --git a/tests/SiteConfiguration.php b/tests/SiteConfiguration.php new file mode 100644 index 0000000..c2e1fdc --- /dev/null +++ b/tests/SiteConfiguration.php @@ -0,0 +1,618 @@ +<?php +/** + * Configuration holder, particularly for multi-wiki sites. + * + * This lives in mediawiki but was copied into the mediawiki-config + * repository to allow tests to run in isolation. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * This is a class for holding configuration settings, particularly for + * multi-wiki sites. + * + * A basic synopsis: + * + * Consider a wikifarm having three sites: two production sites, one in English + * and one in German, and one testing site. You can assign them easy-to-remember + * identifiers - ISO 639 codes 'en' and 'de' for language wikis, and 'beta' for + * the testing wiki. + * + * You would thus initialize the site configuration by specifying the wiki + * identifiers: + * + * @code + * $conf = new SiteConfiguration; + * $conf->wikis = array( 'de', 'en', 'beta' ); + * @endcode + * + * When configuring the MediaWiki global settings (the $wg variables), + * the identifiers will be available to specify settings on a per wiki basis. + * + * @code + * $conf->settings = array( + * 'wgSomeSetting' => array( + * + * # production: + * 'de' => false, + * 'en' => false, + * + * # test: + * 'beta => true, + * ), + * ); + * @endcode + * + * With three wikis, that is easy to manage. But what about a farm with + * hundreds of wikis? Site configuration provides a special keyword named + * 'default' which is the value used when a wiki is not found. Hence + * the above code could be written: + * + * @code + * $conf->settings = array( + * 'wgSomeSetting' => array( + * + * 'default' => false, + * + * # Enable feature on test + * 'beta' => true, + * ), + * ); + * @endcode + * + * + * Since settings can contain arrays, site configuration provides a way + * to merge an array with the default. This is very useful to avoid + * repeating settings again and again while still maintaining specific changes + * on a per wiki basis. + * + * @code + * $conf->settings = array( + * 'wgMergeSetting' = array( + * # Value that will be shared among all wikis: + * 'default' => array( NS_USER => true ), + * + * # Leading '+' means merging the array of value with the defaults + * '+beta' => array( NS_HELP => true ), + * ), + * ); + * + * # Get configuration for the German site: + * $conf->get( 'wgMergeSetting', 'de' ); + * // --> array( NS_USER => true ); + * + * # Get configuration for the testing site: + * $conf->get( 'wgMergeSetting', 'beta' ); + * // --> array( NS_USER => true, NS_HELP => true ); + * @endcode + * + * Finally, to load all configuration settings, extract them in global context: + * + * @code + * # Name / identifier of the wiki as set in $conf->wikis + * $wikiID = 'beta'; + * $globals = $conf->getAll( $wikiID ); + * extract( $globals ); + * @endcode + * + * @todo Give examples for, + * suffixes: + * $conf->suffixes = array( 'wiki' ); + * localVHosts + * callbacks! + */ +class SiteConfiguration { + + /** + * Array of suffixes, for self::siteFromDB() + */ + public $suffixes = array(); + + /** + * Array of wikis, should be the same as $wgLocalDatabases + */ + public $wikis = array(); + + /** + * The whole array of settings + */ + public $settings = array(); + + /** + * Array of domains that are local and can be handled by the same server + * + * @deprecated since 1.25; use $wgLocalVirtualHosts instead. + */ + public $localVHosts = array(); + + /** + * Optional callback to load full configuration data. + * @var string|array + */ + public $fullLoadCallback = null; + + /** Whether or not all data has been loaded */ + public $fullLoadDone = false; + + /** + * A callback function that returns an array with the following keys (all + * optional): + * - suffix: site's suffix + * - lang: site's lang + * - tags: array of wiki tags + * - params: array of parameters to be replaced + * The function will receive the SiteConfiguration instance in the first + * argument and the wiki in the second one. + * if suffix and lang are passed they will be used for the return value of + * self::siteFromDB() and self::$suffixes will be ignored + * + * @var string|array + */ + public $siteParamsCallback = null; + + /** + * Configuration cache for getConfig() + * @var array + */ + protected $cfgCache = array(); + + /** + * Retrieves a configuration setting for a given wiki. + * @param string $settingName ID of the setting name to retrieve + * @param string $wiki Wiki ID of the wiki in question. + * @param string $suffix The suffix of the wiki in question. + * @param array $params List of parameters. $.'key' is replaced by $value in all returned data. + * @param array $wikiTags The tags assigned to the wiki. + * @return mixed The value of the setting requested. + */ + public function get( $settingName, $wiki, $suffix = null, $params = array(), + $wikiTags = array() + ) { + $params = $this->mergeParams( $wiki, $suffix, $params, $wikiTags ); + return $this->getSetting( $settingName, $wiki, $params ); + } + + /** + * Really retrieves a configuration setting for a given wiki. + * + * @param string $settingName ID of the setting name to retrieve. + * @param string $wiki Wiki ID of the wiki in question. + * @param array $params Array of parameters. + * @return mixed The value of the setting requested. + */ + protected function getSetting( $settingName, $wiki, array $params ) { + $retval = null; + if ( array_key_exists( $settingName, $this->settings ) ) { + $thisSetting =& $this->settings[$settingName]; + do { + // Do individual wiki settings + if ( array_key_exists( $wiki, $thisSetting ) ) { + $retval = $thisSetting[$wiki]; + break; + } elseif ( array_key_exists( "+$wiki", $thisSetting ) && is_array( $thisSetting["+$wiki"] ) ) { + $retval = $thisSetting["+$wiki"]; + } + + // Do tag settings + foreach ( $params['tags'] as $tag ) { + if ( array_key_exists( $tag, $thisSetting ) ) { + if ( is_array( $retval ) && is_array( $thisSetting[$tag] ) ) { + $retval = self::arrayMerge( $retval, $thisSetting[$tag] ); + } else { + $retval = $thisSetting[$tag]; + } + break 2; + } elseif ( array_key_exists( "+$tag", $thisSetting ) && is_array( $thisSetting["+$tag"] ) ) { + if ( $retval === null ) { + $retval = array(); + } + $retval = self::arrayMerge( $retval, $thisSetting["+$tag"] ); + } + } + // Do suffix settings + $suffix = $params['suffix']; + if ( !is_null( $suffix ) ) { + if ( array_key_exists( $suffix, $thisSetting ) ) { + if ( is_array( $retval ) && is_array( $thisSetting[$suffix] ) ) { + $retval = self::arrayMerge( $retval, $thisSetting[$suffix] ); + } else { + $retval = $thisSetting[$suffix]; + } + break; + } elseif ( array_key_exists( "+$suffix", $thisSetting ) + && is_array( $thisSetting["+$suffix"] ) + ) { + if ( $retval === null ) { + $retval = array(); + } + $retval = self::arrayMerge( $retval, $thisSetting["+$suffix"] ); + } + } + + // Fall back to default. + if ( array_key_exists( 'default', $thisSetting ) ) { + if ( is_array( $retval ) && is_array( $thisSetting['default'] ) ) { + $retval = self::arrayMerge( $retval, $thisSetting['default'] ); + } else { + $retval = $thisSetting['default']; + } + break; + } + } while ( false ); + } + + if ( !is_null( $retval ) && count( $params['params'] ) ) { + foreach ( $params['params'] as $key => $value ) { + $retval = $this->doReplace( '$' . $key, $value, $retval ); + } + } + return $retval; + } + + /** + * Type-safe string replace; won't do replacements on non-strings + * private? + * + * @param string $from + * @param string $to + * @param string|array $in + * @return string + */ + function doReplace( $from, $to, $in ) { + if ( is_string( $in ) ) { + return str_replace( $from, $to, $in ); + } elseif ( is_array( $in ) ) { + foreach ( $in as $key => $val ) { + $in[$key] = $this->doReplace( $from, $to, $val ); + } + return $in; + } else { + return $in; + } + } + + /** + * Gets all settings for a wiki + * @param string $wiki Wiki ID of the wiki in question. + * @param string $suffix The suffix of the wiki in question. + * @param array $params List of parameters. $.'key' is replaced by $value in all returned data. + * @param array $wikiTags The tags assigned to the wiki. + * @return array Array of settings requested. + */ + public function getAll( $wiki, $suffix = null, $params = array(), $wikiTags = array() ) { + $params = $this->mergeParams( $wiki, $suffix, $params, $wikiTags ); + $localSettings = array(); + foreach ( $this->settings as $varname => $stuff ) { + $append = false; + $var = $varname; + if ( substr( $varname, 0, 1 ) == '+' ) { + $append = true; + $var = substr( $varname, 1 ); + } + + $value = $this->getSetting( $varname, $wiki, $params ); + if ( $append && is_array( $value ) && is_array( $GLOBALS[$var] ) ) { + $value = self::arrayMerge( $value, $GLOBALS[$var] ); + } + if ( !is_null( $value ) ) { + $localSettings[$var] = $value; + } + } + return $localSettings; + } + + /** + * Retrieves a configuration setting for a given wiki, forced to a boolean. + * @param string $setting ID of the setting name to retrieve + * @param string $wiki Wiki ID of the wiki in question. + * @param string $suffix The suffix of the wiki in question. + * @param array $wikiTags The tags assigned to the wiki. + * @return bool The value of the setting requested. + */ + public function getBool( $setting, $wiki, $suffix = null, $wikiTags = array() ) { + return (bool)$this->get( $setting, $wiki, $suffix, array(), $wikiTags ); + } + + /** + * Retrieves an array of local databases + * + * @return array + */ + function &getLocalDatabases() { + return $this->wikis; + } + + /** + * Retrieves the value of a given setting, and places it in a variable passed by reference. + * @param string $setting ID of the setting name to retrieve + * @param string $wiki Wiki ID of the wiki in question. + * @param string $suffix The suffix of the wiki in question. + * @param array $var Reference The variable to insert the value into. + * @param array $params List of parameters. $.'key' is replaced by $value in all returned data. + * @param array $wikiTags The tags assigned to the wiki. + */ + public function extractVar( $setting, $wiki, $suffix, &$var, + $params = array(), $wikiTags = array() + ) { + $value = $this->get( $setting, $wiki, $suffix, $params, $wikiTags ); + if ( !is_null( $value ) ) { + $var = $value; + } + } + + /** + * Retrieves the value of a given setting, and places it in its corresponding global variable. + * @param string $setting ID of the setting name to retrieve + * @param string $wiki Wiki ID of the wiki in question. + * @param string $suffix The suffix of the wiki in question. + * @param array $params List of parameters. $.'key' is replaced by $value in all returned data. + * @param array $wikiTags The tags assigned to the wiki. + */ + public function extractGlobal( $setting, $wiki, $suffix = null, + $params = array(), $wikiTags = array() + ) { + $params = $this->mergeParams( $wiki, $suffix, $params, $wikiTags ); + $this->extractGlobalSetting( $setting, $wiki, $params ); + } + + /** + * @param string $setting + * @param string $wiki + * @param array $params + */ + public function extractGlobalSetting( $setting, $wiki, $params ) { + $value = $this->getSetting( $setting, $wiki, $params ); + if ( !is_null( $value ) ) { + if ( substr( $setting, 0, 1 ) == '+' && is_array( $value ) ) { + $setting = substr( $setting, 1 ); + if ( is_array( $GLOBALS[$setting] ) ) { + $GLOBALS[$setting] = self::arrayMerge( $GLOBALS[$setting], $value ); + } else { + $GLOBALS[$setting] = $value; + } + } else { + $GLOBALS[$setting] = $value; + } + } + } + + /** + * Retrieves the values of all settings, and places them in their corresponding global variables. + * @param string $wiki Wiki ID of the wiki in question. + * @param string $suffix The suffix of the wiki in question. + * @param array $params List of parameters. $.'key' is replaced by $value in all returned data. + * @param array $wikiTags The tags assigned to the wiki. + */ + public function extractAllGlobals( $wiki, $suffix = null, $params = array(), + $wikiTags = array() + ) { + $params = $this->mergeParams( $wiki, $suffix, $params, $wikiTags ); + foreach ( $this->settings as $varName => $setting ) { + $this->extractGlobalSetting( $varName, $wiki, $params ); + } + } + + /** + * Return specific settings for $wiki + * See the documentation of self::$siteParamsCallback for more in-depth + * documentation about this function + * + * @param string $wiki + * @return array + */ + protected function getWikiParams( $wiki ) { + static $default = array( + 'suffix' => null, + 'lang' => null, + 'tags' => array(), + 'params' => array(), + ); + + if ( !is_callable( $this->siteParamsCallback ) ) { + return $default; + } + + $ret = call_user_func_array( $this->siteParamsCallback, array( $this, $wiki ) ); + # Validate the returned value + if ( !is_array( $ret ) ) { + return $default; + } + + foreach ( $default as $name => $def ) { + if ( !isset( $ret[$name] ) || ( is_array( $default[$name] ) && !is_array( $ret[$name] ) ) ) { + $ret[$name] = $default[$name]; + } + } + + return $ret; + } + + /** + * Merge params between the ones passed to the function and the ones given + * by self::$siteParamsCallback for backward compatibility + * Values returned by self::getWikiParams() have the priority. + * + * @param string $wiki Wiki ID of the wiki in question. + * @param string $suffix The suffix of the wiki in question. + * @param array $params List of parameters. $.'key' is replaced by $value in + * all returned data. + * @param array $wikiTags The tags assigned to the wiki. + * @return array + */ + protected function mergeParams( $wiki, $suffix, array $params, array $wikiTags ) { + $ret = $this->getWikiParams( $wiki ); + + if ( is_null( $ret['suffix'] ) ) { + $ret['suffix'] = $suffix; + } + + $ret['tags'] = array_unique( array_merge( $ret['tags'], $wikiTags ) ); + + $ret['params'] += $params; + + // Automatically fill that ones if needed + if ( !isset( $ret['params']['lang'] ) && !is_null( $ret['lang'] ) ) { + $ret['params']['lang'] = $ret['lang']; + } + if ( !isset( $ret['params']['site'] ) && !is_null( $ret['suffix'] ) ) { + $ret['params']['site'] = $ret['suffix']; + } + + return $ret; + } + + /** + * Work out the site and language name from a database name + * @param string $db + * + * @return array + */ + public function siteFromDB( $db ) { + // Allow override + $def = $this->getWikiParams( $db ); + if ( !is_null( $def['suffix'] ) && !is_null( $def['lang'] ) ) { + return array( $def['suffix'], $def['lang'] ); + } + + $site = null; + $lang = null; + foreach ( $this->suffixes as $altSite => $suffix ) { + if ( $suffix === '' ) { + $site = ''; + $lang = $db; + break; + } elseif ( substr( $db, -strlen( $suffix ) ) == $suffix ) { + $site = is_numeric( $altSite ) ? $suffix : $altSite; + $lang = substr( $db, 0, strlen( $db ) - strlen( $suffix ) ); + break; + } + } + $lang = str_replace( '_', '-', $lang ); + return array( $site, $lang ); + } + + /** + * Get the resolved (post-setup) configuration of a potentially foreign wiki. + * For foreign wikis, this is expensive, and only works if maintenance + * scripts are setup to handle the --wiki parameter such as in wiki farms. + * + * @param string $wiki + * @param array|string $settings A setting name or array of setting names + * @return mixed|mixed[] Array if $settings is an array, otherwise the value + * @throws MWException + * @since 1.21 + */ + public function getConfig( $wiki, $settings ) { + global $IP; + + $multi = is_array( $settings ); + $settings = (array)$settings; + if ( $wiki === wfWikiID() ) { // $wiki is this wiki + $res = array(); + foreach ( $settings as $name ) { + if ( !preg_match( '/^wg[A-Z]/', $name ) ) { + throw new MWException( "Variable '$name' does start with 'wg'." ); + } elseif ( !isset( $GLOBALS[$name] ) ) { + throw new MWException( "Variable '$name' is not set." ); + } + $res[$name] = $GLOBALS[$name]; + } + } else { // $wiki is a foreign wiki + if ( isset( $this->cfgCache[$wiki] ) ) { + $res = array_intersect_key( $this->cfgCache[$wiki], array_flip( $settings ) ); + if ( count( $res ) == count( $settings ) ) { + return $multi ? $res : current( $res ); // cache hit + } + } elseif ( !in_array( $wiki, $this->wikis ) ) { + throw new MWException( "No such wiki '$wiki'." ); + } else { + $this->cfgCache[$wiki] = array(); + } + $retVal = 1; + $cmd = wfShellWikiCmd( + "$IP/maintenance/getConfiguration.php", + array( + '--wiki', $wiki, + '--settings', implode( ' ', $settings ), + '--format', 'PHP' + ) + ); + // ulimit5.sh breaks this call + $data = trim( wfShellExec( $cmd, $retVal, array(), array( 'memory' => 0 ) ) ); + if ( $retVal != 0 || !strlen( $data ) ) { + throw new MWException( "Failed to run getConfiguration.php." ); + } + $res = unserialize( $data ); + if ( !is_array( $res ) ) { + throw new MWException( "Failed to unserialize configuration array." ); + } + $this->cfgCache[$wiki] = $this->cfgCache[$wiki] + $res; + } + + return $multi ? $res : current( $res ); + } + + /** + * Returns true if the given vhost is handled locally. + * + * @deprecated since 1.25; check if the host is in $wgLocalVirtualHosts instead. + * @param string $vhost + * @return bool + */ + public function isLocalVHost( $vhost ) { + return in_array( $vhost, $this->localVHosts ); + } + + /** + * Merge multiple arrays together. + * On encountering duplicate keys, merge the two, but ONLY if they're arrays. + * PHP's array_merge_recursive() merges ANY duplicate values into arrays, + * which is not fun + * + * @param array $array1 + * + * @return array + */ + static function arrayMerge( $array1/* ... */ ) { + $out = $array1; + $argsCount = func_num_args(); + for ( $i = 1; $i < $argsCount; $i++ ) { + foreach ( func_get_arg( $i ) as $key => $value ) { + if ( isset( $out[$key] ) && is_array( $out[$key] ) && is_array( $value ) ) { + $out[$key] = self::arrayMerge( $out[$key], $value ); + } elseif ( !isset( $out[$key] ) || !$out[$key] && !is_numeric( $key ) ) { + // Values that evaluate to true given precedence, for the + // primary purpose of merging permissions arrays. + $out[$key] = $value; + } elseif ( is_numeric( $key ) ) { + $out[] = $value; + } + } + } + + return $out; + } + + public function loadFullData() { + if ( $this->fullLoadCallback && !$this->fullLoadDone ) { + call_user_func( $this->fullLoadCallback, $this ); + $this->fullLoadDone = true; + } + } +} diff --git a/tests/bootstrap.php b/tests/bootstrap.php index eafd35d..438824e 100644 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -1,6 +1,6 @@ <?php // Load the shared utilities classes from here! - -require_once( dirname( __FILE__ ) . "/DBList.php" ); -require_once( dirname( __FILE__ ) . "/Provide.php" ); +require_once __DIR__ . "/DBList.php"; +require_once __DIR__ . "/Provide.php"; +require_once __DIR__ . "/Defines.php"; diff --git a/tests/cirrusTest.php b/tests/cirrusTest.php new file mode 100644 index 0000000..7e8b3aa --- /dev/null +++ b/tests/cirrusTest.php @@ -0,0 +1,72 @@ +<?php + +require_once __DIR__ . '/../multiversion/MWMultiVersion.php'; +require_once __DIR__ . '/SiteConfiguration.php'; + +class cirrusTests extends PHPUnit_Framework_TestCase { + public function testClusterConfigurationForProdTestwiki() { + $config = $this->loadCirrusConfig( 'production', 'testwiki', 'wiki', 'en', 'wikipedia' ); + $this->assertArrayNotHasKey( 'wgCirrusSearchServers', $config ); + $this->assertArrayHasKey( 'wgCirrusSearchClusters', $config ); + $this->assertArrayHasKey( 'wgCirrusSearchDefaultCluster', $config ); + $this->assertEquals( 'eqiad', $config['wgCirrusSearchDefaultCluster'] ); + } + + public function testClusterConfigurationForProdEnwiki() { + $config = $this->loadCirrusConfig( 'production', 'enwiki', 'wiki', 'en', 'wikipedia' ); + $this->assertArrayHasKey( 'wgCirrusSearchServers', $config ); + $this->assertArrayHasKey( 'wgCirrusSearchClusters', $config ); + $this->assertCount( 1, $config['wgCirrusSearchClusters'] ); + $this->assertEquals( + $config['wgCirrusSearchServers'], + reset( $config['wgCirrusSearchClusters'] ) + ); + $this->assertEquals( + $config['wgCirrusSearchDefaultCluster'], + reset( array_keys( $config['wgCirrusSearchClusters'] ) ) + ); + } + + private function loadCirrusConfig( $wmfRealm, $wgDBname, $dbSuffix, $lang, $site ) { + // Variables rqeuired for wgConf.php + $wmfConfigDir = __DIR__ . "/../wmf-config"; + $IP = __DIR__ .'/../php/'; + + require "{$wmfConfigDir}/wgConf.php"; + + // InitialiseSettings.php explicitly declares these as global, so we must too + $GLOBALS['wmfUdp2logDest'] = 'localhost'; + $GLOBALS['wmfDatacenter'] = 'unittest'; + $GLOBALS['wmfRealm'] = $wmfRealm; + $GLOBALS['wmfConfigDir'] = $wmfConfigDir; + $GLOBALS['wgConf'] = $wgConf; + + require "{$wmfConfigDir}/InitialiseSettings.php"; + + $globals = $wgConf->getAll( $wgDBname, $dbSuffix, array( + 'lang' => $lang, + 'docRoot' => '/dev/null', + 'site' => $site, + 'stdlogo' => 'file://dev/null', + ), array() ); + + extract( $globals ); + + // variables that would have been setup elsewhere, perhaps in mediawiki + // default settings or by CommonSettings.php, or by CirrusSearch.php, + // but none of those are a part of this repository + $wgJobTypeConf = array( 'default' => array() ); + $wgCirrusSearchWeights = array(); + $wgCirrusSearchNamespaceWeights = array(); + $wmfSwiftEqiadConfig = array( + 'cirrusAuthUrl' => '', + 'cirrusUser' => '', + 'cirrusKey' => '', + ); + $wgCirrusSearchPoolCounterKey = 'unittest:poolcounter:blahblahblah'; + + require "{$wmfConfigDir}/CirrusSearch-common.php"; + + return compact( array_keys( get_defined_vars() ) ); + } +} diff --git a/tests/dbconfigTest.php b/tests/dbconfigTest.php index 34cb912..4127f11 100644 --- a/tests/dbconfigTest.php +++ b/tests/dbconfigTest.php @@ -8,7 +8,7 @@ * @file */ -require __DIR__ . '/../multiversion/MWRealm.php'; +require_once __DIR__ . '/../multiversion/MWRealm.php'; class dbconfigTests extends PHPUnit_Framework_TestCase { diff --git a/wmf-config/CirrusSearch-common.php b/wmf-config/CirrusSearch-common.php index b301b95..5e419f5 100644 --- a/wmf-config/CirrusSearch-common.php +++ b/wmf-config/CirrusSearch-common.php @@ -11,8 +11,6 @@ # # Contact Wikimedia operations or platform engineering for more details. -require_once( "$IP/extensions/Elastica/Elastica.php" ); -require_once( "$IP/extensions/CirrusSearch/CirrusSearch.php" ); $wgSearchType = 'CirrusSearch'; if ( $wmgUseClusterJobqueue ) { @@ -23,6 +21,11 @@ $wgJobTypeConf['default']; } +if ( isset( $wmgCirrusSearchServers ) ) { + $wgCirrusSearchServers = $wmgCirrusSearchServers; +} +$wgCirrusSearchClusters = $wmgCirrusSearchClusters; +$wgCirrusSearchDefaultCluster = $wmgCirrusSearchDefaultCluster; # Enable user testing $wgCirrusSearchUserTesting = $wmgCirrusSearchUserTesting; diff --git a/wmf-config/CirrusSearch-labs.php b/wmf-config/CirrusSearch-labs.php index 0970dfb..f5edb48 100644 --- a/wmf-config/CirrusSearch-labs.php +++ b/wmf-config/CirrusSearch-labs.php @@ -5,13 +5,6 @@ # to the 'labs' realm which in most of the cases means the beta cluster. # It should be loaded AFTER CirrusSearch-common.php -$wgCirrusSearchServers = array( - 'deployment-elastic05', - 'deployment-elastic06', - 'deployment-elastic07', - 'deployment-elastic08', -); - if ( $wgDBname == 'enwiki' ) { $wgCirrusSearchInterwikiSources = array( 'wiktionary' => 'enwiktionary', diff --git a/wmf-config/CirrusSearch-production.php b/wmf-config/CirrusSearch-production.php index a5287e1..1bb20f7 100644 --- a/wmf-config/CirrusSearch-production.php +++ b/wmf-config/CirrusSearch-production.php @@ -5,9 +5,6 @@ # to the 'production' realm. # It should be loaded AFTER CirrusSearch-common.php -$wgCirrusSearchServers = array( - '10.2.2.30', # search.svc.eqiad.wmnet -); $wgCirrusSearchConnectionAttempts = 3; diff --git a/wmf-config/CommonSettings.php b/wmf-config/CommonSettings.php index 937d515..e11ca74 100755 --- a/wmf-config/CommonSettings.php +++ b/wmf-config/CommonSettings.php @@ -897,6 +897,8 @@ # All wikis are special and get Cirrus :) if ( $wmgUseCirrus ) { + require_once( "$IP/extensions/Elastica/Elastica.php" ); + require_once( "$IP/extensions/CirrusSearch/CirrusSearch.php" ); include( "$wmfConfigDir/CirrusSearch-common.php" ); } diff --git a/wmf-config/InitialiseSettings-labs.php b/wmf-config/InitialiseSettings-labs.php index 9c18a05..55d1d2d 100644 --- a/wmf-config/InitialiseSettings-labs.php +++ b/wmf-config/InitialiseSettings-labs.php @@ -955,6 +955,21 @@ 'zhwiki' => true, ), + '-wmgCirrusSearchServers' => array( + 'default' => null, + ), + + '-wmgCirrusSearchClusters' => array( + 'default' => array( + 'eqiad' => array( + 'deployment-elastic05', + 'deployment-elastic06', + 'deployment-elastic07', + 'deployment-elastic08', + ), + ), + ), + 'wmgUseFlow' => array( 'enwiki' => true, 'en_rtlwiki' => true, diff --git a/wmf-config/InitialiseSettings.php b/wmf-config/InitialiseSettings.php index 20ad4ce..07cb3ea 100644 --- a/wmf-config/InitialiseSettings.php +++ b/wmf-config/InitialiseSettings.php @@ -15281,6 +15281,25 @@ 'default' => true, ), +'wmgCirrusSearchServers' => array( + 'default' => array( '10.2.2.30' ), // search.svc.eqiad.wmnet + 'testwiki' => null, +), + +'wmgCirrusSearchDefaultCluster' => array( + 'default' => 'eqiad', +), + +'wmgCirrusSearchClusters' => array( + 'default' => array( + 'eqiad' => array( '10.2.2.30' ), // search.svc.eqiad.wmnet + ), + 'testwiki' => array( + 'eqiad' => array( '10.2.2.30' ), // search.svc.eqiad.wmnet + 'labsearch' => array( 'nobelium.eqiad.wmnet' ), + ), +), + 'wmgCirrusPerUserPoolCounter' => array( 'default' => false, 'testwiki' => true, -- To view, visit https://gerrit.wikimedia.org/r/244077 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I3566d40bf93f5766d0825bb63f9804550ae19d33 Gerrit-PatchSet: 1 Gerrit-Project: operations/mediawiki-config Gerrit-Branch: master Gerrit-Owner: EBernhardson <ebernhard...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
