Tim Starling has submitted this change and it was merged.
Change subject: [JobQueue] Use target wiki configuration for some key functions.
......................................................................
[JobQueue] Use target wiki configuration for some key functions.
* getQueueTypes() now gets the config of the target wiki.
Previously, for WMF, if an extension using jobs was not
on aawiki, those job queues would not be seen by nextJobDB.
* Also fixed nextJobDB.php so that it no longer only considers
jobs types known to aawiki for picking a DB for default jobs.
* Note that $wgJobTypesExcludedFromDefaultQueue should be global.
* This adds a SiteConfiguration::getConfig() function, which calls
a new getConfiguration.php script.
Change-Id: I7e6904ead17efa407291f423a2b18e3c866d55fd
---
M includes/SiteConfiguration.php
M includes/job/JobQueueGroup.php
A maintenance/getConfiguration.php
M maintenance/nextJobDB.php
4 files changed, 183 insertions(+), 6 deletions(-)
Approvals:
Tim Starling: Verified; Looks good to me, approved
diff --git a/includes/SiteConfiguration.php b/includes/SiteConfiguration.php
index 7e96d45..11d7fd6 100644
--- a/includes/SiteConfiguration.php
+++ b/includes/SiteConfiguration.php
@@ -162,6 +162,12 @@
public $siteParamsCallback = null;
/**
+ * Configuration cache for getConfig()
+ * @var array
+ */
+ protected $cfgCache = array();
+
+ /**
* Retrieves a configuration setting for a given wiki.
* @param $settingName String ID of the setting name to retrieve
* @param $wiki String Wiki ID of the wiki in question.
@@ -487,6 +493,67 @@
}
/**
+ * Get the resolved (post-setup) configuration of a potentially foreign
wiki.
+ * For foreign wikis, this is expensive, and only works if maintenance
+ * scripts are setup to handle the --wiki parameter such as in wiki
farms.
+ *
+ * @param string $wiki
+ * @param array|string $settings A setting name or array of setting
names
+ * @return Array|mixed Array if $settings is an array, otherwise the
value
+ * @throws MWException
+ * @since 1.21
+ */
+ public function getConfig( $wiki, $settings ) {
+ global $IP;
+
+ $multi = is_array( $settings );
+ $settings = (array)$settings;
+ if ( $wiki === wfWikiID() ) { // $wiki is this wiki
+ $res = array();
+ foreach ( $settings as $name ) {
+ if ( !preg_match( '/^wg[A-Z]/', $name ) ) {
+ throw new MWException( "Variable
'$name' does start with 'wg'." );
+ } elseif ( !isset( $GLOBALS[$name] ) ) {
+ throw new MWException( "Variable
'$name' is not set." );
+ }
+ $res[$name] = $GLOBALS[$name];
+ }
+ } else { // $wiki is a foreign wiki
+ if ( isset( $this->cfgCache[$wiki] ) ) {
+ $res = array_intersect_key(
$this->cfgCache[$wiki], array_flip( $settings ) );
+ if ( count( $res ) == count( $settings ) ) {
+ return $res; // cache hit
+ }
+ } elseif ( !in_array( $wiki, $this->wikis ) ) {
+ throw new MWException( "No such wiki '$wiki'."
);
+ } else {
+ $this->cfgCache[$wiki] = array();
+ }
+ $retVal = 1;
+ $cmd = wfShellWikiCmd(
+ "$IP/maintenance/getConfiguration.php",
+ array(
+ '--wiki', $wiki,
+ '--settings', implode( ' ', $settings ),
+ '--format', 'PHP'
+ )
+ );
+ // ulimit5.sh breaks this call
+ $data = trim( wfShellExec( $cmd, $retVal, array(),
array( 'memory' => 0 ) ) );
+ if ( $retVal != 0 || !strlen( $data ) ) {
+ throw new MWException( "Failed to run
getConfiguration.php." );
+ }
+ $res = unserialize( $data );
+ if ( !is_array( $res ) ) {
+ throw new MWException( "Failed to unserialize
configuration array." );
+ }
+ $this->cfgCache[$wiki] = $this->cfgCache[$wiki] + $res;
+ }
+
+ return $multi ? $res : current( $res );
+ }
+
+ /**
* Returns true if the given vhost is handled locally.
* @param $vhost String
* @return bool
diff --git a/includes/job/JobQueueGroup.php b/includes/job/JobQueueGroup.php
index 0118853..32c881f 100644
--- a/includes/job/JobQueueGroup.php
+++ b/includes/job/JobQueueGroup.php
@@ -200,9 +200,7 @@
* @return array List of strings
*/
public function getQueueTypes() {
- global $wgJobClasses;
-
- return array_keys( $wgJobClasses );
+ return array_keys( $this->getCachedConfigVar( 'wgJobClasses' )
);
}
/**
@@ -284,4 +282,23 @@
return $count;
}
+
+ private function getCachedConfigVar( $name ) {
+ global $wgConf, $wgMemc;
+
+ if ( $this->wiki === wfWikiID() ) {
+ return $GLOBALS[$name]; // common case
+ } else {
+ list( $db, $prefix ) = wfSplitWikiID( $this->wiki );
+ $key = wfForeignMemcKey( $db, $prefix, 'configvalue',
$name );
+ $value = $wgMemc->get( $key ); // ('v' => ...) or false
+ if ( is_array( $value ) ) {
+ return $value['v'];
+ } else {
+ $value = $wgConf->getConfig( $this->wiki, $name
);
+ $wgMemc->set( $key, array( 'v' => $value ),
86400 + mt_rand( 0, 86400 ) );
+ return $value;
+ }
+ }
+ }
}
diff --git a/maintenance/getConfiguration.php b/maintenance/getConfiguration.php
new file mode 100644
index 0000000..83b5b02
--- /dev/null
+++ b/maintenance/getConfiguration.php
@@ -0,0 +1,89 @@
+<?php
+/**
+ * Print serialized output of MediaWiki config vars
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Maintenance
+ * @author Tim Starling
+ * @author Antoine Musso
+ */
+
+require_once( __DIR__ . '/Maintenance.php' );
+
+/**
+ * Print serialized output of MediaWiki config vars
+ *
+ * @ingroup Maintenance
+ */
+class GetConfiguration extends Maintenance {
+ public function __construct() {
+ parent::__construct();
+ $this->mDescription = "Get serialized MediaWiki site
configuration";
+ $this->addOption( 'settings', 'Space-separated list of wg*
variables', true, true );
+ $this->addOption( 'format', 'PHP or JSON', true, true );
+ $this->addOption( 'wiki', 'Wiki ID', true, true );
+ }
+
+ public function execute() {
+ $res = array();
+ foreach ( explode( ' ', $this->getOption( 'settings' ) ) as
$name ) {
+ if ( !preg_match( '/^wg[A-Z]/', $name ) ) {
+ throw new MWException( "Variable '$name' does
start with 'wg'." );
+ } elseif ( !isset( $GLOBALS[$name] ) ) {
+ throw new MWException( "Variable '$name' is not
set." );
+ } elseif ( !$this->isAllowedVariable( $GLOBALS[$name] )
) {
+ throw new MWException( "Variable '$name'
includes non-array, non-scalar, items." );
+ }
+ $res[$name] = $GLOBALS[$name];
+ }
+
+ $out = null;
+ switch( $this->getOption( 'format' ) ) {
+ case 'PHP':
+ $out = serialize( $res );
+ break;
+ case 'JSON':
+ $out = FormatJson::encode( $res );
+ break;
+ default:
+ throw new MWException( "Invalid serialization
format given." );
+ }
+ if ( !is_string( $out ) ) {
+ throw new MWException( "Failed to serialize the
requested settings." );
+ }
+
+ $this->output( $out . "\n" );
+ }
+
+ private function isAllowedVariable( $value ) {
+ if ( is_array( $value ) ) {
+ foreach ( $value as $k => $v ) {
+ if ( !$this->isAllowedVariable( $v ) ) {
+ return false;
+ }
+ }
+ return true;
+ } elseif ( is_scalar( $value ) ) {
+ return true;
+ }
+ return false;
+ }
+}
+
+$maintClass = "GetConfiguration";
+require_once( RUN_MAINTENANCE_IF_MAIN );
diff --git a/maintenance/nextJobDB.php b/maintenance/nextJobDB.php
index cf90cf6..6cc8566 100644
--- a/maintenance/nextJobDB.php
+++ b/maintenance/nextJobDB.php
@@ -37,15 +37,16 @@
}
public function execute() {
- global $wgMemc;
+ global $wgMemc, $wgJobTypesExcludedFromDefaultQueue;
$type = false; // job type required/picked
+
if ( $this->hasOption( 'types' ) ) {
$types = explode( ' ', $this->getOption( 'types' ) );
} elseif ( $this->hasOption( 'type' ) ) {
$types = array( $this->getOption( 'type' ) );
} else {
- $types =
JobQueueGroup::singleton()->getDefaultQueueTypes();
+ $types = false;
}
// Handle any required periodic queue maintenance
@@ -64,7 +65,10 @@
// Flatten the tree of candidates into a flat list so
that a random
// item can be selected, weighing each queue (type/db
tuple) equally.
foreach ( $pendingDBs as $type => $dbs ) {
- if ( in_array( $type, $types ) ) {
+ if (
+ ( is_array( $types ) && in_array(
$type, $types ) ) ||
+ ( $types === false && !in_array( $type,
$wgJobTypesExcludedFromDefaultQueue ) )
+ ) {
foreach ( $dbs as $db ) {
$candidates[] = array( $type,
$db );
}
--
To view, visit https://gerrit.wikimedia.org/r/47378
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I7e6904ead17efa407291f423a2b18e3c866d55fd
Gerrit-PatchSet: 6
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Aaron Schulz <[email protected]>
Gerrit-Reviewer: Aaron Schulz <[email protected]>
Gerrit-Reviewer: Lwelling <[email protected]>
Gerrit-Reviewer: Tim Starling <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
