Tim Starling has submitted this change and it was merged. Change subject: [JobQueue] Use target wiki configuration for some key functions. ......................................................................
[JobQueue] Use target wiki configuration for some key functions. * getQueueTypes() now gets the config of the target wiki. Previously, for WMF, if an extension using jobs was not on aawiki, those job queues would not be seen by nextJobDB. * Also fixed nextJobDB.php so that it no longer only considers jobs types known to aawiki for picking a DB for default jobs. * Note that $wgJobTypesExcludedFromDefaultQueue should be global. * This adds a SiteConfiguration::getConfig() function, which calls a new getConfiguration.php script. Change-Id: I7e6904ead17efa407291f423a2b18e3c866d55fd --- M includes/SiteConfiguration.php M includes/job/JobQueueGroup.php A maintenance/getConfiguration.php M maintenance/nextJobDB.php 4 files changed, 183 insertions(+), 6 deletions(-) Approvals: Tim Starling: Verified; Looks good to me, approved diff --git a/includes/SiteConfiguration.php b/includes/SiteConfiguration.php index 7e96d45..11d7fd6 100644 --- a/includes/SiteConfiguration.php +++ b/includes/SiteConfiguration.php @@ -162,6 +162,12 @@ public $siteParamsCallback = null; /** + * Configuration cache for getConfig() + * @var array + */ + protected $cfgCache = array(); + + /** * Retrieves a configuration setting for a given wiki. * @param $settingName String ID of the setting name to retrieve * @param $wiki String Wiki ID of the wiki in question. @@ -487,6 +493,67 @@ } /** + * Get the resolved (post-setup) configuration of a potentially foreign wiki. + * For foreign wikis, this is expensive, and only works if maintenance + * scripts are setup to handle the --wiki parameter such as in wiki farms. + * + * @param string $wiki + * @param array|string $settings A setting name or array of setting names + * @return Array|mixed Array if $settings is an array, otherwise the value + * @throws MWException + * @since 1.21 + */ + public function getConfig( $wiki, $settings ) { + global $IP; + + $multi = is_array( $settings ); + $settings = (array)$settings; + if ( $wiki === wfWikiID() ) { // $wiki is this wiki + $res = array(); + foreach ( $settings as $name ) { + if ( !preg_match( '/^wg[A-Z]/', $name ) ) { + throw new MWException( "Variable '$name' does start with 'wg'." ); + } elseif ( !isset( $GLOBALS[$name] ) ) { + throw new MWException( "Variable '$name' is not set." ); + } + $res[$name] = $GLOBALS[$name]; + } + } else { // $wiki is a foreign wiki + if ( isset( $this->cfgCache[$wiki] ) ) { + $res = array_intersect_key( $this->cfgCache[$wiki], array_flip( $settings ) ); + if ( count( $res ) == count( $settings ) ) { + return $res; // cache hit + } + } elseif ( !in_array( $wiki, $this->wikis ) ) { + throw new MWException( "No such wiki '$wiki'." ); + } else { + $this->cfgCache[$wiki] = array(); + } + $retVal = 1; + $cmd = wfShellWikiCmd( + "$IP/maintenance/getConfiguration.php", + array( + '--wiki', $wiki, + '--settings', implode( ' ', $settings ), + '--format', 'PHP' + ) + ); + // ulimit5.sh breaks this call + $data = trim( wfShellExec( $cmd, $retVal, array(), array( 'memory' => 0 ) ) ); + if ( $retVal != 0 || !strlen( $data ) ) { + throw new MWException( "Failed to run getConfiguration.php." ); + } + $res = unserialize( $data ); + if ( !is_array( $res ) ) { + throw new MWException( "Failed to unserialize configuration array." ); + } + $this->cfgCache[$wiki] = $this->cfgCache[$wiki] + $res; + } + + return $multi ? $res : current( $res ); + } + + /** * Returns true if the given vhost is handled locally. * @param $vhost String * @return bool diff --git a/includes/job/JobQueueGroup.php b/includes/job/JobQueueGroup.php index 0118853..32c881f 100644 --- a/includes/job/JobQueueGroup.php +++ b/includes/job/JobQueueGroup.php @@ -200,9 +200,7 @@ * @return array List of strings */ public function getQueueTypes() { - global $wgJobClasses; - - return array_keys( $wgJobClasses ); + return array_keys( $this->getCachedConfigVar( 'wgJobClasses' ) ); } /** @@ -284,4 +282,23 @@ return $count; } + + private function getCachedConfigVar( $name ) { + global $wgConf, $wgMemc; + + if ( $this->wiki === wfWikiID() ) { + return $GLOBALS[$name]; // common case + } else { + list( $db, $prefix ) = wfSplitWikiID( $this->wiki ); + $key = wfForeignMemcKey( $db, $prefix, 'configvalue', $name ); + $value = $wgMemc->get( $key ); // ('v' => ...) or false + if ( is_array( $value ) ) { + return $value['v']; + } else { + $value = $wgConf->getConfig( $this->wiki, $name ); + $wgMemc->set( $key, array( 'v' => $value ), 86400 + mt_rand( 0, 86400 ) ); + return $value; + } + } + } } diff --git a/maintenance/getConfiguration.php b/maintenance/getConfiguration.php new file mode 100644 index 0000000..83b5b02 --- /dev/null +++ b/maintenance/getConfiguration.php @@ -0,0 +1,89 @@ +<?php +/** + * Print serialized output of MediaWiki config vars + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + * @author Tim Starling + * @author Antoine Musso + */ + +require_once( __DIR__ . '/Maintenance.php' ); + +/** + * Print serialized output of MediaWiki config vars + * + * @ingroup Maintenance + */ +class GetConfiguration extends Maintenance { + public function __construct() { + parent::__construct(); + $this->mDescription = "Get serialized MediaWiki site configuration"; + $this->addOption( 'settings', 'Space-separated list of wg* variables', true, true ); + $this->addOption( 'format', 'PHP or JSON', true, true ); + $this->addOption( 'wiki', 'Wiki ID', true, true ); + } + + public function execute() { + $res = array(); + foreach ( explode( ' ', $this->getOption( 'settings' ) ) as $name ) { + if ( !preg_match( '/^wg[A-Z]/', $name ) ) { + throw new MWException( "Variable '$name' does start with 'wg'." ); + } elseif ( !isset( $GLOBALS[$name] ) ) { + throw new MWException( "Variable '$name' is not set." ); + } elseif ( !$this->isAllowedVariable( $GLOBALS[$name] ) ) { + throw new MWException( "Variable '$name' includes non-array, non-scalar, items." ); + } + $res[$name] = $GLOBALS[$name]; + } + + $out = null; + switch( $this->getOption( 'format' ) ) { + case 'PHP': + $out = serialize( $res ); + break; + case 'JSON': + $out = FormatJson::encode( $res ); + break; + default: + throw new MWException( "Invalid serialization format given." ); + } + if ( !is_string( $out ) ) { + throw new MWException( "Failed to serialize the requested settings." ); + } + + $this->output( $out . "\n" ); + } + + private function isAllowedVariable( $value ) { + if ( is_array( $value ) ) { + foreach ( $value as $k => $v ) { + if ( !$this->isAllowedVariable( $v ) ) { + return false; + } + } + return true; + } elseif ( is_scalar( $value ) ) { + return true; + } + return false; + } +} + +$maintClass = "GetConfiguration"; +require_once( RUN_MAINTENANCE_IF_MAIN ); diff --git a/maintenance/nextJobDB.php b/maintenance/nextJobDB.php index cf90cf6..6cc8566 100644 --- a/maintenance/nextJobDB.php +++ b/maintenance/nextJobDB.php @@ -37,15 +37,16 @@ } public function execute() { - global $wgMemc; + global $wgMemc, $wgJobTypesExcludedFromDefaultQueue; $type = false; // job type required/picked + if ( $this->hasOption( 'types' ) ) { $types = explode( ' ', $this->getOption( 'types' ) ); } elseif ( $this->hasOption( 'type' ) ) { $types = array( $this->getOption( 'type' ) ); } else { - $types = JobQueueGroup::singleton()->getDefaultQueueTypes(); + $types = false; } // Handle any required periodic queue maintenance @@ -64,7 +65,10 @@ // Flatten the tree of candidates into a flat list so that a random // item can be selected, weighing each queue (type/db tuple) equally. foreach ( $pendingDBs as $type => $dbs ) { - if ( in_array( $type, $types ) ) { + if ( + ( is_array( $types ) && in_array( $type, $types ) ) || + ( $types === false && !in_array( $type, $wgJobTypesExcludedFromDefaultQueue ) ) + ) { foreach ( $dbs as $db ) { $candidates[] = array( $type, $db ); } -- To view, visit https://gerrit.wikimedia.org/r/47378 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I7e6904ead17efa407291f423a2b18e3c866d55fd Gerrit-PatchSet: 6 Gerrit-Project: mediawiki/core Gerrit-Branch: master Gerrit-Owner: Aaron Schulz <asch...@wikimedia.org> Gerrit-Reviewer: Aaron Schulz <asch...@wikimedia.org> Gerrit-Reviewer: Lwelling <lwell...@wikimedia.org> Gerrit-Reviewer: Tim Starling <tstarl...@wikimedia.org> Gerrit-Reviewer: jenkins-bot _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits