jenkins-bot has submitted this change and it was merged.
Change subject: Move LoadMonitorMySQL to LoadMonitor
......................................................................
Move LoadMonitorMySQL to LoadMonitor
* The LoadMonitor interface is now ILoadMonitor.
* Nothing in this code was MySQL specific.
Now any DB type can benefit from a LoadMonitor.
Change-Id: I272a72cd55a70f99a866d518d44cb3bcaca91b9e
---
M autoload.php
M includes/libs/rdbms/loadbalancer/LoadBalancer.php
A includes/libs/rdbms/loadmonitor/ILoadMonitor.php
M includes/libs/rdbms/loadmonitor/LoadMonitor.php
M includes/libs/rdbms/loadmonitor/LoadMonitorMySQL.php
M includes/libs/rdbms/loadmonitor/LoadMonitorNull.php
6 files changed, 201 insertions(+), 167 deletions(-)
Approvals:
Legoktm: Looks good to me, approved
jenkins-bot: Verified
diff --git a/autoload.php b/autoload.php
index 9fd83eb..1cb71d8 100644
--- a/autoload.php
+++ b/autoload.php
@@ -583,6 +583,7 @@
'IExpiringStore' => __DIR__ .
'/includes/libs/objectcache/IExpiringStore.php',
'IJobSpecification' => __DIR__ .
'/includes/jobqueue/JobSpecification.php',
'ILoadBalancer' => __DIR__ .
'/includes/libs/rdbms/loadbalancer/ILoadBalancer.php',
+ 'ILoadMonitor' => __DIR__ .
'/includes/libs/rdbms/loadmonitor/ILoadMonitor.php',
'IP' => __DIR__ . '/includes/utils/IP.php',
'IPSet' => __DIR__ . '/includes/compat/IPSetCompat.php',
'IPTC' => __DIR__ . '/includes/media/IPTC.php',
diff --git a/includes/libs/rdbms/loadbalancer/LoadBalancer.php
b/includes/libs/rdbms/loadbalancer/LoadBalancer.php
index cea7523..36279bc 100644
--- a/includes/libs/rdbms/loadbalancer/LoadBalancer.php
+++ b/includes/libs/rdbms/loadbalancer/LoadBalancer.php
@@ -45,7 +45,7 @@
/** @var array[] $aliases Map of (table => (dbname, schema, prefix)
map) */
private $tableAliases = [];
- /** @var LoadMonitor */
+ /** @var ILoadMonitor */
private $mLoadMonitor;
/** @var BagOStuff */
private $srvCache;
@@ -190,7 +190,7 @@
/**
* Get a LoadMonitor instance
*
- * @return LoadMonitor
+ * @return ILoadMonitor
*/
private function getLoadMonitor() {
if ( !isset( $this->mLoadMonitor ) ) {
diff --git a/includes/libs/rdbms/loadmonitor/ILoadMonitor.php
b/includes/libs/rdbms/loadmonitor/ILoadMonitor.php
new file mode 100644
index 0000000..e355c03
--- /dev/null
+++ b/includes/libs/rdbms/loadmonitor/ILoadMonitor.php
@@ -0,0 +1,65 @@
+<?php
+/**
+ * Database load monitoring interface
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Database
+ */
+use Psr\Log\LoggerAwareInterface;
+
+/**
+ * An interface for database load monitoring
+ *
+ * @ingroup Database
+ */
+interface ILoadMonitor extends LoggerAwareInterface {
+ /**
+ * Construct a new LoadMonitor with a given LoadBalancer parent
+ *
+ * @param ILoadBalancer $lb LoadBalancer this instance serves
+ * @param BagOStuff $sCache Local server memory cache
+ * @param BagOStuff $cCache Local cluster memory cache
+ */
+ public function __construct( ILoadBalancer $lb, BagOStuff $sCache,
BagOStuff $cCache );
+
+ /**
+ * Perform pre-connection load ratio adjustment.
+ * @param int[] &$loads
+ * @param string|bool $group The selected query group. Default: false
+ * @param string|bool $domain Default: false
+ */
+ public function scaleLoads( &$loads, $group = false, $domain = false );
+
+ /**
+ * Get an estimate of replication lag (in seconds) for each server
+ *
+ * Values may be "false" if replication is too broken to estimate
+ *
+ * @param integer[] $serverIndexes
+ * @param string $domain
+ *
+ * @return array Map of (server index => float|int|bool)
+ */
+ public function getLagTimes( $serverIndexes, $domain );
+
+ /**
+ * Clear any process and persistent cache of lag times
+ * @since 1.27
+ */
+ public function clearCaches();
+}
diff --git a/includes/libs/rdbms/loadmonitor/LoadMonitor.php
b/includes/libs/rdbms/loadmonitor/LoadMonitor.php
index 460746b..1da8f4e 100644
--- a/includes/libs/rdbms/loadmonitor/LoadMonitor.php
+++ b/includes/libs/rdbms/loadmonitor/LoadMonitor.php
@@ -1,7 +1,5 @@
<?php
/**
- * Database load monitoring.
- *
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
@@ -20,46 +18,140 @@
* @file
* @ingroup Database
*/
-use Psr\Log\LoggerAwareInterface;
+
+use Psr\Log\LoggerInterface;
/**
- * An interface for database load monitoring
+ * Basic DB load monitor with no external dependencies
+ * Uses memcached to cache the replication lag for a short time
*
* @ingroup Database
*/
-interface LoadMonitor extends LoggerAwareInterface {
- /**
- * Construct a new LoadMonitor with a given LoadBalancer parent
- *
- * @param ILoadBalancer $lb LoadBalancer this instance serves
- * @param BagOStuff $sCache Local server memory cache
- * @param BagOStuff $cCache Local cluster memory cache
- */
- public function __construct( ILoadBalancer $lb, BagOStuff $sCache,
BagOStuff $cCache );
+class LoadMonitor implements ILoadMonitor {
+ /** @var ILoadBalancer */
+ protected $parent;
+ /** @var BagOStuff */
+ protected $srvCache;
+ /** @var BagOStuff */
+ protected $mainCache;
+ /** @var LoggerInterface */
+ protected $replLogger;
- /**
- * Perform pre-connection load ratio adjustment.
- * @param int[] &$loads
- * @param string|bool $group The selected query group. Default: false
- * @param string|bool $domain Default: false
- */
- public function scaleLoads( &$loads, $group = false, $domain = false );
+ public function __construct( ILoadBalancer $lb, BagOStuff $srvCache,
BagOStuff $cache ) {
+ $this->parent = $lb;
+ $this->srvCache = $srvCache;
+ $this->mainCache = $cache;
+ $this->replLogger = new \Psr\Log\NullLogger();
+ }
- /**
- * Get an estimate of replication lag (in seconds) for each server
- *
- * Values may be "false" if replication is too broken to estimate
- *
- * @param integer[] $serverIndexes
- * @param string $domain
- *
- * @return array Map of (server index => float|int|bool)
- */
- public function getLagTimes( $serverIndexes, $domain );
+ public function setLogger( LoggerInterface $logger ) {
+ $this->replLogger = $logger;
+ }
- /**
- * Clear any process and persistent cache of lag times
- * @since 1.27
- */
- public function clearCaches();
+ public function scaleLoads( &$loads, $group = false, $domain = false ) {
+ }
+
+ public function getLagTimes( $serverIndexes, $domain ) {
+ if ( count( $serverIndexes ) == 1 && reset( $serverIndexes ) ==
0 ) {
+ # Single server only, just return zero without caching
+ return [ 0 => 0 ];
+ }
+
+ $key = $this->getLagTimeCacheKey();
+ # Randomize TTLs to reduce stampedes (4.0 - 5.0 sec)
+ $ttl = mt_rand( 4e6, 5e6 ) / 1e6;
+ # Keep keys around longer as fallbacks
+ $staleTTL = 60;
+
+ # (a) Check the local APC cache
+ $value = $this->srvCache->get( $key );
+ if ( $value && $value['timestamp'] > ( microtime( true ) - $ttl
) ) {
+ $this->replLogger->debug( __METHOD__ . ": got lag times
($key) from local cache" );
+ return $value['lagTimes']; // cache hit
+ }
+ $staleValue = $value ?: false;
+
+ # (b) Check the shared cache and backfill APC
+ $value = $this->mainCache->get( $key );
+ if ( $value && $value['timestamp'] > ( microtime( true ) - $ttl
) ) {
+ $this->srvCache->set( $key, $value, $staleTTL );
+ $this->replLogger->debug( __METHOD__ . ": got lag times
($key) from main cache" );
+
+ return $value['lagTimes']; // cache hit
+ }
+ $staleValue = $value ?: $staleValue;
+
+ # (c) Cache key missing or expired; regenerate and backfill
+ if ( $this->mainCache->lock( $key, 0, 10 ) ) {
+ # Let this process alone update the cache value
+ $cache = $this->mainCache;
+ /** @noinspection PhpUnusedLocalVariableInspection */
+ $unlocker = new ScopedCallback( function () use (
$cache, $key ) {
+ $cache->unlock( $key );
+ } );
+ } elseif ( $staleValue ) {
+ # Could not acquire lock but an old cache exists, so
use it
+ return $staleValue['lagTimes'];
+ }
+
+ $lagTimes = [];
+ foreach ( $serverIndexes as $i ) {
+ if ( $i == $this->parent->getWriterIndex() ) {
+ $lagTimes[$i] = 0; // master always has no lag
+ continue;
+ }
+
+ $conn = $this->parent->getAnyOpenConnection( $i );
+ if ( $conn ) {
+ $close = false; // already open
+ } else {
+ $conn = $this->parent->openConnection( $i,
$domain );
+ $close = true; // new connection
+ }
+
+ if ( !$conn ) {
+ $lagTimes[$i] = false;
+ $host = $this->parent->getServerName( $i );
+ $this->replLogger->error( __METHOD__ . ": host
$host (#$i) is unreachable" );
+ continue;
+ }
+
+ $lagTimes[$i] = $conn->getLag();
+ if ( $lagTimes[$i] === false ) {
+ $host = $this->parent->getServerName( $i );
+ $this->replLogger->error( __METHOD__ . ": host
$host (#$i) is not replicating?" );
+ }
+
+ if ( $close ) {
+ # Close the connection to avoid sleeper
connections piling up.
+ # Note that the caller will pick one of these
DBs and reconnect,
+ # which is slightly inefficient, but this only
matters for the lag
+ # time cache miss cache, which is far less
common that cache hits.
+ $this->parent->closeConnection( $conn );
+ }
+ }
+
+ # Add a timestamp key so we know when it was cached
+ $value = [ 'lagTimes' => $lagTimes, 'timestamp' => microtime(
true ) ];
+ $this->mainCache->set( $key, $value, $staleTTL );
+ $this->srvCache->set( $key, $value, $staleTTL );
+ $this->replLogger->info( __METHOD__ . ": re-calculated lag
times ($key)" );
+
+ return $value['lagTimes'];
+ }
+
+ public function clearCaches() {
+ $key = $this->getLagTimeCacheKey();
+ $this->srvCache->delete( $key );
+ $this->mainCache->delete( $key );
+ }
+
+ private function getLagTimeCacheKey() {
+ $writerIndex = $this->parent->getWriterIndex();
+ // Lag is per-server, not per-DB, so key on the master DB name
+ return $this->srvCache->makeGlobalKey(
+ 'lag-times',
+ $this->parent->getServerName( $writerIndex )
+ );
+ }
}
diff --git a/includes/libs/rdbms/loadmonitor/LoadMonitorMySQL.php
b/includes/libs/rdbms/loadmonitor/LoadMonitorMySQL.php
index 02babfe..7286417 100644
--- a/includes/libs/rdbms/loadmonitor/LoadMonitorMySQL.php
+++ b/includes/libs/rdbms/loadmonitor/LoadMonitorMySQL.php
@@ -19,139 +19,15 @@
* @ingroup Database
*/
-use Psr\Log\LoggerInterface;
-
/**
* Basic MySQL load monitor with no external dependencies
* Uses memcached to cache the replication lag for a short time
*
* @ingroup Database
*/
-class LoadMonitorMySQL implements LoadMonitor {
- /** @var ILoadBalancer */
- protected $parent;
- /** @var BagOStuff */
- protected $srvCache;
- /** @var BagOStuff */
- protected $mainCache;
- /** @var LoggerInterface */
- protected $replLogger;
-
- public function __construct( ILoadBalancer $lb, BagOStuff $srvCache,
BagOStuff $cache ) {
- $this->parent = $lb;
- $this->srvCache = $srvCache;
- $this->mainCache = $cache;
- $this->replLogger = new \Psr\Log\NullLogger();
- }
-
- public function setLogger( LoggerInterface $logger ) {
- $this->replLogger = $logger;
- }
-
- public function scaleLoads( &$loads, $group = false, $wiki = false ) {
- }
-
- public function getLagTimes( $serverIndexes, $wiki ) {
- if ( count( $serverIndexes ) == 1 && reset( $serverIndexes ) ==
0 ) {
- # Single server only, just return zero without caching
- return [ 0 => 0 ];
- }
-
- $key = $this->getLagTimeCacheKey();
- # Randomize TTLs to reduce stampedes (4.0 - 5.0 sec)
- $ttl = mt_rand( 4e6, 5e6 ) / 1e6;
- # Keep keys around longer as fallbacks
- $staleTTL = 60;
-
- # (a) Check the local APC cache
- $value = $this->srvCache->get( $key );
- if ( $value && $value['timestamp'] > ( microtime( true ) - $ttl
) ) {
- $this->replLogger->debug( __METHOD__ . ": got lag times
($key) from local cache" );
- return $value['lagTimes']; // cache hit
- }
- $staleValue = $value ?: false;
-
- # (b) Check the shared cache and backfill APC
- $value = $this->mainCache->get( $key );
- if ( $value && $value['timestamp'] > ( microtime( true ) - $ttl
) ) {
- $this->srvCache->set( $key, $value, $staleTTL );
- $this->replLogger->debug( __METHOD__ . ": got lag times
($key) from main cache" );
-
- return $value['lagTimes']; // cache hit
- }
- $staleValue = $value ?: $staleValue;
-
- # (c) Cache key missing or expired; regenerate and backfill
- if ( $this->mainCache->lock( $key, 0, 10 ) ) {
- # Let this process alone update the cache value
- $cache = $this->mainCache;
- /** @noinspection PhpUnusedLocalVariableInspection */
- $unlocker = new ScopedCallback( function () use (
$cache, $key ) {
- $cache->unlock( $key );
- } );
- } elseif ( $staleValue ) {
- # Could not acquire lock but an old cache exists, so
use it
- return $staleValue['lagTimes'];
- }
-
- $lagTimes = [];
- foreach ( $serverIndexes as $i ) {
- if ( $i == $this->parent->getWriterIndex() ) {
- $lagTimes[$i] = 0; // master always has no lag
- continue;
- }
-
- $conn = $this->parent->getAnyOpenConnection( $i );
- if ( $conn ) {
- $close = false; // already open
- } else {
- $conn = $this->parent->openConnection( $i,
$wiki );
- $close = true; // new connection
- }
-
- if ( !$conn ) {
- $lagTimes[$i] = false;
- $host = $this->parent->getServerName( $i );
- $this->replLogger->error( __METHOD__ . ": host
$host (#$i) is unreachable" );
- continue;
- }
-
- $lagTimes[$i] = $conn->getLag();
- if ( $lagTimes[$i] === false ) {
- $host = $this->parent->getServerName( $i );
- $this->replLogger->error( __METHOD__ . ": host
$host (#$i) is not replicating?" );
- }
-
- if ( $close ) {
- # Close the connection to avoid sleeper
connections piling up.
- # Note that the caller will pick one of these
DBs and reconnect,
- # which is slightly inefficient, but this only
matters for the lag
- # time cache miss cache, which is far less
common that cache hits.
- $this->parent->closeConnection( $conn );
- }
- }
-
- # Add a timestamp key so we know when it was cached
- $value = [ 'lagTimes' => $lagTimes, 'timestamp' => microtime(
true ) ];
- $this->mainCache->set( $key, $value, $staleTTL );
- $this->srvCache->set( $key, $value, $staleTTL );
- $this->replLogger->info( __METHOD__ . ": re-calculated lag
times ($key)" );
-
- return $value['lagTimes'];
- }
-
- public function clearCaches() {
- $key = $this->getLagTimeCacheKey();
- $this->srvCache->delete( $key );
- $this->mainCache->delete( $key );
- }
-
- private function getLagTimeCacheKey() {
- $writerIndex = $this->parent->getWriterIndex();
- // Lag is per-server, not per-DB, so key on the master DB name
- return $this->srvCache->makeGlobalKey(
- 'lag-times',
- $this->parent->getServerName( $writerIndex )
- );
+class LoadMonitorMySQL extends LoadMonitor {
+ public function scaleLoads( &$loads, $group = false, $domain = false ) {
+ // @TODO: maybe use Threads_running/Threads_created ratio to
guess load
+ // and Queries/Uptime to guess if a server is warming up the
buffer pool
}
}
diff --git a/includes/libs/rdbms/loadmonitor/LoadMonitorNull.php
b/includes/libs/rdbms/loadmonitor/LoadMonitorNull.php
index 1a40b8f..8062001 100644
--- a/includes/libs/rdbms/loadmonitor/LoadMonitorNull.php
+++ b/includes/libs/rdbms/loadmonitor/LoadMonitorNull.php
@@ -20,7 +20,7 @@
*/
use Psr\Log\LoggerInterface;
-class LoadMonitorNull implements LoadMonitor {
+class LoadMonitorNull implements ILoadMonitor {
public function __construct( ILoadBalancer $lb, BagOStuff $sCache,
BagOStuff $cCache ) {
}
--
To view, visit https://gerrit.wikimedia.org/r/310733
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I272a72cd55a70f99a866d518d44cb3bcaca91b9e
Gerrit-PatchSet: 5
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Aaron Schulz <asch...@wikimedia.org>
Gerrit-Reviewer: Legoktm <legoktm.wikipe...@gmail.com>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
