Aaron Schulz has uploaded a new change for review.
https://gerrit.wikimedia.org/r/232899
Change subject: [WIP] Support async writes to secondary MultiWriteBagOStuff
stores
......................................................................
[WIP] Support async writes to secondary MultiWriteBagOStuff stores
* This is useful for ParserCache, as it tries to focus on memcached
and use other caches (e.g. mariadb) for the long-tail of less used
content, as setup on WMF. The class uses BagOStuff in a way that is
compatible with this approach.
Bug: T109751
Change-Id: Ia64eb44a9b52a988fde27b468d604d9163bed4b4
---
M includes/objectcache/MultiWriteBagOStuff.php
M includes/parser/ParserCache.php
2 files changed, 60 insertions(+), 14 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/99/232899/1
diff --git a/includes/objectcache/MultiWriteBagOStuff.php
b/includes/objectcache/MultiWriteBagOStuff.php
index b5f3bd9..eac0d9e 100644
--- a/includes/objectcache/MultiWriteBagOStuff.php
+++ b/includes/objectcache/MultiWriteBagOStuff.php
@@ -31,26 +31,52 @@
class MultiWriteBagOStuff extends BagOStuff {
/** @var BagOStuff[] */
protected $caches;
+ /** @var bool Use async secondary writes */
+ protected $asyncReplication = false;
+ /** @var array[] */
+ protected $asyncWrites = array();
/**
- * Constructor. Parameters are:
- *
- * - caches: This should have a numbered array of cache parameter
- * structures, in the style required by $wgObjectCaches.
See
- * the documentation of $wgObjectCaches for more detail.
+ * $params include:
+ * - caches: This should have a numbered array of cache parameter
+ * structures, in the style required by
$wgObjectCaches. See
+ * the documentation of $wgObjectCaches for more
detail.
+ * The first cache is the primary one, being the first
to
+ * be read in the fallback chain. Writes happen to all
stores
+ * in the order they are defined. However,
lock()/unlock() calls
+ * only use the primary store.
+ * - replication: Either 'sync' or 'async'. This controls whether
writes to
+ * secondary stores are deferred when possible. Async
writes
+ * require the HHVM register_postsend_function()
function.
+ * Async writes can increase the chance of some race
conditions
+ * or cause keys to expire seconds later than
expected. It is
+ * safe to use for modules when cached values: are
immutable,
+ * invalidation uses logical TTLs, invalidation uses
etag/timestamp
+ * validation against the DB, or merge() is used to
handle races.
*
* @param array $params
* @throws InvalidArgumentException
*/
public function __construct( $params ) {
parent::__construct( $params );
+
if ( !isset( $params['caches'] ) ) {
- throw new InvalidArgumentException( __METHOD__ . ': the
caches parameter is required' );
+ throw new InvalidArgumentException( __METHOD__ . ':
"caches" parameter required' );
}
$this->caches = array();
foreach ( $params['caches'] as $cacheInfo ) {
$this->caches[] = ObjectCache::newFromParams(
$cacheInfo );
+ }
+
+ if (
+ isset( $params['replication'] ) &&
+ $params['replication'] === 'async' &&
+ PHP_SAPI !== 'cli' &&
+ function_exists( 'register_postsend_function' )
+ ) {
+ $this->asyncReplication = true;
+ register_postsend_function( array( $this,
'doAsyncWritesInternal' ) );
}
}
@@ -174,11 +200,18 @@
$args = func_get_args();
array_shift( $args );
- foreach ( $this->caches as $cache ) {
- if ( !call_user_func_array( array( $cache, $method ),
$args ) ) {
- $ret = false;
+ foreach ( $this->caches as $i => $cache ) {
+ if ( $i == 0 || !$this->asyncReplication ) {
+ // First store or in sync mode: write now and
get result
+ if ( !call_user_func_array( array( $cache,
$method ), $args ) ) {
+ $ret = false;
+ }
+ } else {
+ // Secondary write in async mode: do not block
this HTTP request
+ $this->asyncWrites[] = array( $cache, $method,
$args );
}
}
+
return $ret;
}
@@ -199,4 +232,18 @@
}
return $ret;
}
+
+ /**
+ * Execute all async writes still pending in FIFO order
+ */
+ public function doAsyncWritesInternal() {
+ foreach ( $this->asyncWrites as $write ) {
+ list( $cache, $method, $args ) = $write;
+ if ( !call_user_func_array( array( $cache, $method ),
$args ) ) {
+ $this->logger->warning( "Async $method op
failed" );
+ }
+ }
+
+ $this->asyncWrites = array();
+ }
}
diff --git a/includes/parser/ParserCache.php b/includes/parser/ParserCache.php
index 47fcd30..abff543 100644
--- a/includes/parser/ParserCache.php
+++ b/includes/parser/ParserCache.php
@@ -44,15 +44,14 @@
/**
* Setup a cache pathway with a given back-end storage mechanism.
- * May be a memcached client or a BagOStuff derivative.
+ *
+ * This class use an invalidation strategy that is compatible with
+ * MultiWriteBagOStuff in async replication mode.
*
* @param BagOStuff $memCached
* @throws MWException
*/
- protected function __construct( $memCached ) {
- if ( !$memCached ) {
- throw new MWException( "Tried to create a ParserCache
with an invalid memcached" );
- }
+ protected function __construct( BagOStuff $memCached ) {
$this->mMemc = $memCached;
}
--
To view, visit https://gerrit.wikimedia.org/r/232899
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ia64eb44a9b52a988fde27b468d604d9163bed4b4
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Aaron Schulz <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
