jenkins-bot has submitted this change and it was merged.
Change subject: objectcache: add WANObjectCache::getMultiWithSetCallback()
......................................................................
objectcache: add WANObjectCache::getMultiWithSetCallback()
This does what it says on the tin, e.g. a way to fetch
multiple keys at once. Callbacks are still called in a
serial manner when needed.
Change-Id: I8e24a6de7f46499a53ec41636c5a4f106b9b3d09
---
M includes/libs/objectcache/WANObjectCache.php
M tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php
2 files changed, 274 insertions(+), 8 deletions(-)
Approvals:
Ori.livneh: Looks good to me, approved
jenkins-bot: Verified
diff --git a/includes/libs/objectcache/WANObjectCache.php
b/includes/libs/objectcache/WANObjectCache.php
index 5f6e324..d7db732 100644
--- a/includes/libs/objectcache/WANObjectCache.php
+++ b/includes/libs/objectcache/WANObjectCache.php
@@ -88,6 +88,9 @@
/** @var int ERR_* constant for the "last error" registry */
protected $lastRelayError = self::ERR_NONE;
+ /** @var mixed[] Temporary warm-up cache */
+ private $warmupCache = [];
+
/** Max time expected to pass between delete() and DB commit finishing
*/
const MAX_COMMIT_DELAY = 3;
/** Max replication+snapshot lag before applying TTL_LAGGED or
disallowing set() */
@@ -284,7 +287,14 @@
}
// Fetch all of the raw values
- $wrappedValues = $this->cache->getMulti( array_merge(
$valueKeys, $checkKeysFlat ) );
+ $keysGet = array_merge( $valueKeys, $checkKeysFlat );
+ if ( $this->warmupCache ) {
+ $wrappedValues = array_intersect_key(
$this->warmupCache, array_flip( $keysGet ) );
+ $keysGet = array_diff( $keysGet, array_keys(
$wrappedValues ) ); // keys left to fetch
+ } else {
+ $wrappedValues = [];
+ }
+ $wrappedValues += $this->cache->getMulti( $keysGet );
// Time used to compare/init "check" keys (derived after
getMulti() to be pessimistic)
$now = microtime( true );
@@ -1017,6 +1027,95 @@
}
/**
+ * Method to fetch/regenerate multiple cache keys at once
+ *
+ * This works the same as getWithSetCallback() except:
+ * - a) The $keys argument expects the result of
WANObjectCache::makeMultiKeys()
+ * - b) The $callback argument expects a callback taking the
following arguments:
+ * - $id: ID of an entity to query
+ * - $oldValue : the prior cache value or false if none was
present
+ * - &$ttl : a reference to the new value TTL in seconds
+ * - &$setOpts : a reference to options for set() which can be
altered
+ * - $oldAsOf : generation UNIX timestamp of $oldValue or null
if not present
+ * Aside from the additional $id argument, the other arguments
function the same
+ * way they do in getWithSetCallback().
+ * - c) The return value is a map of (cache key => value) in the
order of $keyedIds
+ *
+ * @see WANObjectCache::getWithSetCallback()
+ *
+ * Example usage:
+ * @code
+ * $rows = $cache->getMultiWithSetCallback(
+ * // Map of cache keys to entitiy IDs
+ * $cache->makeMultiKeys(
+ * $this->fileVersionIds(),
+ * function ( $id, WANObjectCache $cache ) {
+ * return $cache->makeKey( 'file-version', $id );
+ * }
+ * ),
+ * // Time-to-live (in seconds)
+ * $cache::TTL_DAY,
+ * // Function that derives the new key value
+ * return function ( $id, $oldValue, &$ttl, array &$setOpts ) {
+ * $dbr = wfGetDB( DB_REPLICA );
+ * // Account for any snapshot/replica DB lag
+ * $setOpts += Database::getCacheSetOptions( $dbr );
+ *
+ * // Load the row for this file
+ * $row = $dbr->selectRow( 'file', '*', [ 'id' => $id ],
__METHOD__ );
+ *
+ * return $row ? (array)$row : false;
+ * },
+ * [
+ * // Process cache for 30 seconds
+ * 'pcTTL' => 30,
+ * // Use a dedicated 500 item cache (initialized
on-the-fly)
+ * 'pcGroup' => 'file-versions:500'
+ * ]
+ * );
+ * $files = array_map( [ __CLASS__, 'newFromRow' ], $rows );
+ * @endcode
+ *
+ * @param ArrayIterator $keyedIds Result of
WANObjectCache::makeMultiKeys()
+ * @param integer $ttl Seconds to live for key updates
+ * @param callable $callback Callback the yields entity regeneration
callbacks
+ * @param array $opts Options map
+ * @return array Map of (cache key => value) in the same order as
$keyedIds
+ * @since 1.28
+ */
+ final public function getMultiWithSetCallback(
+ ArrayIterator $keyedIds, $ttl, callable $callback, array $opts
= []
+ ) {
+ $keysWarmUp = iterator_to_array( $keyedIds, true );
+ $checkKeys = isset( $opts['checkKeys'] ) ? $opts['checkKeys'] :
[];
+ foreach ( $checkKeys as $i => $checkKeyOrKeys ) {
+ if ( is_int( $i ) ) {
+ $keysWarmUp[] = $checkKeyOrKeys;
+ } else {
+ $keysWarmUp = array_merge( $keysWarmUp,
$checkKeyOrKeys );
+ }
+ }
+
+ $this->warmupCache = $this->cache->getMulti( $keysWarmUp );
+ $this->warmupCache += array_fill_keys( $keysWarmUp, false );
+
+ // Wrap $callback to match the getWithSetCallback() format
while passing $id to $callback
+ $id = null;
+ $func = function ( $oldValue, &$ttl, array $setOpts, $oldAsOf )
use ( $callback, &$id ) {
+ return $callback( $id, $oldValue, $ttl, $setOpts,
$oldAsOf );
+ };
+
+ $values = [];
+ foreach ( $keyedIds as $key => $id ) {
+ $values[$key] = $this->getWithSetCallback( $key, $ttl,
$func, $opts );
+ }
+
+ $this->warmupCache = [];
+
+ return $values;
+ }
+
+ /**
* @see BagOStuff::makeKey()
* @param string ... Key component
* @return string
@@ -1037,6 +1136,21 @@
}
/**
+ * @param array $entities List of entity IDs
+ * @param callable $keyFunc Callback yielding a key from (entity ID,
this WANObjectCache)
+ * @return ArrayIterator Iterator yielding (cache key => entity ID) in
$entities order
+ * @since 1.28
+ */
+ public function makeMultiKeys( array $entities, callable $keyFunc ) {
+ $map = [];
+ foreach ( $entities as $entity ) {
+ $map[$keyFunc( $entity, $this )] = $entity;
+ }
+
+ return new ArrayIterator( $map );
+ }
+
+ /**
* Get the "last error" registered; clearLastError() should be called
manually
* @return int ERR_* class constant for the "last error" registry
*/
diff --git a/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php
b/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php
index 99b959b..f43a3f3 100644
--- a/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php
+++ b/tests/phpunit/includes/libs/objectcache/WANObjectCacheTest.php
@@ -22,6 +22,7 @@
}
$wanCache = TestingAccessWrapper::newFromObject( $this->cache );
+ /** @noinspection PhpUndefinedFieldInspection */
$this->internalCache = $wanCache->cache;
}
@@ -29,13 +30,14 @@
* @dataProvider provideSetAndGet
* @covers WANObjectCache::set()
* @covers WANObjectCache::get()
+ * @covers WANObjectCache::makeKey()
* @param mixed $value
* @param integer $ttl
*/
public function testSetAndGet( $value, $ttl ) {
$curTTL = null;
$asOf = null;
- $key = wfRandomString();
+ $key = $this->cache->makeKey( 'x', wfRandomString() );
$this->cache->get( $key, $curTTL, [], $asOf );
$this->assertNull( $curTTL, "Current TTL is null" );
@@ -71,9 +73,10 @@
/**
* @covers WANObjectCache::get()
+ * @covers WANObjectCache::makeGlobalKey()
*/
public function testGetNotExists() {
- $key = wfRandomString();
+ $key = $this->cache->makeGlobalKey( 'y', wfRandomString(), 'p'
);
$curTTL = null;
$value = $this->cache->get( $key, $curTTL );
@@ -165,7 +168,7 @@
$priorAsOf = null;
$wasSet = 0;
$func = function( $old, &$ttl, &$opts, $asOf )
- use ( &$wasSet, &$priorValue, &$priorAsOf, $value )
+ use ( &$wasSet, &$priorValue, &$priorAsOf, $value )
{
++$wasSet;
$priorValue = $old;
@@ -188,9 +191,9 @@
$wasSet = 0;
$v = $cache->getWithSetCallback( $key, 30, $func, [
- 'lowTTL' => 0,
- 'lockTSE' => 5,
- ] + $extOpts );
+ 'lowTTL' => 0,
+ 'lockTSE' => 5,
+ ] + $extOpts );
$this->assertEquals( $value, $v, "Value returned" );
$this->assertEquals( 0, $wasSet, "Value not regenerated" );
@@ -241,6 +244,150 @@
}
public static function getWithSetCallback_provider() {
+ return [
+ [ [], false ],
+ [ [ 'version' => 1 ], true ]
+ ];
+ }
+
+ /**
+ * @dataProvider getMultiWithSetCallback_provider
+ * @covers WANObjectCache::geMultitWithSetCallback()
+ * @covers WANObjectCache::makeMultiKeys()
+ * @param array $extOpts
+ * @param bool $versioned
+ */
+ public function testGetMultiWithSetCallback( array $extOpts, $versioned
) {
+ $cache = $this->cache;
+
+ $keyA = wfRandomString();
+ $keyB = wfRandomString();
+ $keyC = wfRandomString();
+ $cKey1 = wfRandomString();
+ $cKey2 = wfRandomString();
+
+ $priorValue = null;
+ $priorAsOf = null;
+ $wasSet = 0;
+ $genFunc = function ( $id, $old, &$ttl, &$opts, $asOf ) use (
+ &$wasSet, &$priorValue, &$priorAsOf
+ ) {
+ ++$wasSet;
+ $priorValue = $old;
+ $priorAsOf = $asOf;
+ $ttl = 20; // override with another value
+ return "@$id$";
+ };
+
+ $wasSet = 0;
+ $keyedIds = new ArrayIterator( [ $keyA => 3353 ] );
+ $value = "@3353$";
+ $v = $cache->getMultiWithSetCallback(
+ $keyedIds, 30, $genFunc, [ 'lockTSE' => 5 ] + $extOpts
);
+ $this->assertEquals( $value, $v[$keyA], "Value returned" );
+ $this->assertEquals( 1, $wasSet, "Value regenerated" );
+ $this->assertFalse( $priorValue, "No prior value" );
+ $this->assertNull( $priorAsOf, "No prior value" );
+
+ $curTTL = null;
+ $cache->get( $keyA, $curTTL );
+ $this->assertLessThanOrEqual( 20, $curTTL, 'Current TTL between
19-20 (overriden)' );
+ $this->assertGreaterThanOrEqual( 19, $curTTL, 'Current TTL
between 19-20 (overriden)' );
+
+ $wasSet = 0;
+ $value = "@efef$";
+ $keyedIds = new ArrayIterator( [ $keyB => 'efef' ] );
+ $v = $cache->getMultiWithSetCallback(
+ $keyedIds, 30, $genFunc, [ 'lowTTL' => 0, 'lockTSE' =>
5, ] + $extOpts );
+ $this->assertEquals( $value, $v[$keyB], "Value returned" );
+ $this->assertEquals( 1, $wasSet, "Value regenerated" );
+ $v = $cache->getMultiWithSetCallback(
+ $keyedIds, 30, $genFunc, [ 'lowTTL' => 0, 'lockTSE' =>
5, ] + $extOpts );
+ $this->assertEquals( $value, $v[$keyB], "Value returned" );
+ $this->assertEquals( 1, $wasSet, "Value not regenerated" );
+
+ $priorTime = microtime( true );
+ usleep( 1 );
+ $wasSet = 0;
+ $keyedIds = new ArrayIterator( [ $keyB => 'efef' ] );
+ $v = $cache->getMultiWithSetCallback(
+ $keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1,
$cKey2 ] ] + $extOpts
+ );
+ $this->assertEquals( $value, $v[$keyB], "Value returned" );
+ $this->assertEquals( 1, $wasSet, "Value regenerated due to
check keys" );
+ $this->assertEquals( $value, $priorValue, "Has prior value" );
+ $this->assertType( 'float', $priorAsOf, "Has prior value" );
+ $t1 = $cache->getCheckKeyTime( $cKey1 );
+ $this->assertGreaterThanOrEqual( $priorTime, $t1, 'Check keys
generated on miss' );
+ $t2 = $cache->getCheckKeyTime( $cKey2 );
+ $this->assertGreaterThanOrEqual( $priorTime, $t2, 'Check keys
generated on miss' );
+
+ $priorTime = microtime( true );
+ $value = "@43636$";
+ $wasSet = 0;
+ $keyedIds = new ArrayIterator( [ $keyC => 43636 ] );
+ $v = $cache->getMultiWithSetCallback(
+ $keyedIds, 30, $genFunc, [ 'checkKeys' => [ $cKey1,
$cKey2 ] ] + $extOpts
+ );
+ $this->assertEquals( $value, $v[$keyC], "Value returned" );
+ $this->assertEquals( 1, $wasSet, "Value regenerated due to
still-recent check keys" );
+ $t1 = $cache->getCheckKeyTime( $cKey1 );
+ $this->assertLessThanOrEqual( $priorTime, $t1, 'Check keys did
not change again' );
+ $t2 = $cache->getCheckKeyTime( $cKey2 );
+ $this->assertLessThanOrEqual( $priorTime, $t2, 'Check keys did
not change again' );
+
+ $curTTL = null;
+ $v = $cache->get( $keyC, $curTTL, [ $cKey1, $cKey2 ] );
+ if ( $versioned ) {
+ $this->assertEquals( $value, $v[$cache::VFLD_DATA],
"Value returned" );
+ } else {
+ $this->assertEquals( $value, $v, "Value returned" );
+ }
+ $this->assertLessThanOrEqual( 0, $curTTL, "Value has current
TTL < 0 due to check keys" );
+
+ $wasSet = 0;
+ $key = wfRandomString();
+ $keyedIds = new ArrayIterator( [ $key => 242424 ] );
+ $v = $cache->getMultiWithSetCallback(
+ $keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts );
+ $this->assertEquals( "@{$keyedIds[$key]}$", $v[$key], "Value
returned" );
+ $cache->delete( $key );
+ $keyedIds = new ArrayIterator( [ $key => 242424 ] );
+ $v = $cache->getMultiWithSetCallback(
+ $keyedIds, 30, $genFunc, [ 'pcTTL' => 5 ] + $extOpts );
+ $this->assertEquals( "@{$keyedIds[$key]}$", $v[$key], "Value
still returned after deleted" );
+ $this->assertEquals( 1, $wasSet, "Value process cached while
deleted" );
+
+ $calls = 0;
+ $ids = [ 1, 2, 3, 4, 5, 6 ];
+ $keyFunc = function ( $id, WANObjectCache $wanCache ) {
+ return $wanCache->makeKey( 'test', $id );
+ };
+ $keyedIds = $cache->makeMultiKeys( $ids, $keyFunc );
+ $genFunc = function ( $id, $oldValue, &$ttl, array &$setops )
use ( &$calls ) {
+ ++$calls;
+
+ return "val-{$id}";
+ };
+ $values = $cache->getMultiWithSetCallback( $keyedIds, 10,
$genFunc );
+
+ $this->assertEquals(
+ [ "val-1", "val-2", "val-3", "val-4", "val-5", "val-6"
],
+ array_values( $values ),
+ "Correct values in correct order"
+ );
+ $this->assertEquals(
+ array_map( $keyFunc, $ids, array_fill( 0, count( $ids
), $this->cache ) ),
+ array_keys( $values ),
+ "Correct keys in correct order"
+ );
+ $this->assertEquals( count( $ids ), $calls );
+
+ $cache->getMultiWithSetCallback( $keyedIds, 10, $genFunc );
+ $this->assertEquals( count( $ids ), $calls, "Values cached" );
+ }
+
+ public static function getMultiWithSetCallback_provider() {
return [
[ [], false ],
[ [ 'version' => 1 ], true ]
@@ -777,9 +924,14 @@
/**
* @dataProvider provideAdaptiveTTL
* @covers WANObjectCache::adaptiveTTL()
+ * @param float|int $ago
+ * @param int $maxTTL
+ * @param int $minTTL
+ * @param float $factor
+ * @param int $adaptiveTTL
*/
public function testAdaptiveTTL( $ago, $maxTTL, $minTTL, $factor,
$adaptiveTTL ) {
- $mtime = is_int( $ago ) ? time() - $ago : $ago;
+ $mtime = $ago ? time() - $ago : $ago;
$margin = 5;
$ttl = $this->cache->adaptiveTTL( $mtime, $maxTTL, $minTTL,
$factor );
--
To view, visit https://gerrit.wikimedia.org/r/310248
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I8e24a6de7f46499a53ec41636c5a4f106b9b3d09
Gerrit-PatchSet: 13
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Aaron Schulz <[email protected]>
Gerrit-Reviewer: Ori.livneh <[email protected]>
Gerrit-Reviewer: Sbisson <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
