Aaron Schulz has uploaded a new change for review.
https://gerrit.wikimedia.org/r/309412
Change subject: Handle API request errors and improve caching
......................................................................
Handle API request errors and improve caching
* Use adaptive TTLs so cache entries that change more often have
lower TTLs than those that change less often.
* Also use "minAsOf" together with getChronologyProtectorTouched()
to opportunistically purge the cache. This triggers when a user
makes changes to one of the shared wiki DBs and then hits this
API.
* Increase the TTL to 1 hour, given the above.
Depends-On: Ia1168cf0d46cfdee046838ce4c5a6294e4d81760
Change-Id: Id71bef631888ba3b12b9db59bd62bc2fe3647ea8
---
M ApiFileAnnotations.php
1 file changed, 106 insertions(+), 25 deletions(-)
git pull
ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/FileAnnotations
refs/changes/12/309412/1
diff --git a/ApiFileAnnotations.php b/ApiFileAnnotations.php
index d50a063..8c65ad1 100644
--- a/ApiFileAnnotations.php
+++ b/ApiFileAnnotations.php
@@ -23,13 +23,11 @@
* @copyright 2015 Mark Holmquist
* @license GNU General Public License version 2.0
*/
+use MediaWiki\MediaWikiServices;
class ApiFileAnnotations extends ApiQueryBase {
- // 5 minutes - long enough to avoid crashing the servers with a lot
- // of repeated requests for the same data, but not long enough so it's
- // hard to update information quickly. Cache not invalidated by changes
- // to Wikidata, Wikipedia, or Commons.
- const CACHE_TTL = 300;
+ const MIN_CACHE_TTL = WANObjectCache::TTL_MINUTE;
+ const MAX_CACHE_TTL = WANObjectCache::TTL_HOUR;
public function __construct( $query, $moduleName ) {
parent::__construct( $query, $moduleName, 'fa' );
@@ -86,12 +84,16 @@
protected function renderCommonsAnnotation( $commonsMatches ) {
$categoryName = $commonsMatches[1];
+ $safeAsOf = $this->getSafeCacheAsOfForUser( 'commonswiki' );
+
$cache = ObjectCache::getMainWANInstance();
+ $cacheKey = $cache->makeKey( 'fileannotations',
'commonscategory', $categoryName );
return $cache->getWithSetCallback(
- $cache->makeKey( 'fileannotations', 'commonscategory',
$categoryName ),
- self::CACHE_TTL,
- function ( $oldValue, &$ttl, array &$setOpts ) use (
$categoryName ) {
+ $cacheKey,
+ self::MAX_CACHE_TTL,
+ function ( $oldValue, &$ttl, array &$setOpts, $oldAsOf )
+ use ( $cache, $categoryName, $cacheKey, $safeAsOf ) {
$client = new MultiHttpClient( [] );
$response = $client->run( [
@@ -111,9 +113,14 @@
],
] );
- $imagesApiData = json_decode(
$response['body'], true );
+ if ( $response['code'] == 200 ) {
+ $imagesApiData = json_decode(
$response['body'], true );
+ $pages =
$imagesApiData['query']['pages'];
+ } else {
+ $pages = [];
- $pages = $imagesApiData['query']['pages'];
+ $ttl = $cache::TTL_UNCACHEABLE;
+ }
$imagesHtml = '<div class="category-members">';
@@ -136,25 +143,39 @@
? '<a href="' . $href . '">' . 'See
more images' . '</a>'
: '';
- return
+ $html =
'<div
class="commons-category-annotation">' .
$imagesHtml .
$seeMoreHtml .
'</div>';
- }
+
+ if ( $safeAsOf && $oldValue && $oldValue !==
$html ) {
+ $cache->delete( $cacheKey ); // update
all datacenters
+ $ttl = $cache::TTL_UNCACHEABLE; //
don't bother due to delete() tombstone
+ } else {
+ $virtualMtime = ( $oldValue === $html )
? $oldAsOf : false;
+ $ttl = $cache->adaptiveTTL(
$virtualMtime, $ttl, self::MIN_CACHE_TTL );
+ }
+
+ return $html;
+ },
+ [ 'minAsOf' => $safeAsOf ]
);
}
protected function renderWikipediaAnnotation( $wpMatches ) {
$articleName = $wpMatches[2];
$language = $wpMatches[1];
+ $safeAsOf = $this->getSafeCacheAsOfForUser( 'enwiki' );
$cache = ObjectCache::getMainWANInstance();
+ $cacheKey = $cache->makeKey( 'fileannotations',
'wikipediapage', $language, $articleName );
return $cache->getWithSetCallback(
- $cache->makeKey( 'fileannotations', 'wikipediapage',
$language, $articleName ),
- self::CACHE_TTL,
- function ( $oldValue, &$ttl, array &$setOpts ) use (
$articleName, $language ) {
+ $cacheKey,
+ self::MAX_CACHE_TTL,
+ function ( $oldValue, &$ttl, array &$setOpts, $oldAsOf )
+ use ( $cache, $articleName, $language, $cacheKey,
$safeAsOf ) {
$client = new MultiHttpClient( [] );
$response = $client->run( [
@@ -171,13 +192,18 @@
],
] );
- $articleApiData = json_decode(
$response['body'], true );
+ if ( $response['code'] == 200 ) {
+ $articleApiData = json_decode(
$response['body'], true );
+ $pages =
$articleApiData['query']['pages'];
+ } else {
+ $pages = [];
- $pages = $articleApiData['query']['pages'];
+ $ttl = $cache::TTL_UNCACHEABLE;
+ }
$page = reset( $pages );
// There's only one page, so just do it here
- return
+ $html =
'<div
class="wikipedia-article-annotation">' .
$page['extract'] .
'<p class="pageimage">' .
@@ -190,20 +216,34 @@
'" />' .
'</p>' .
'</div>';
- }
+
+ if ( $safeAsOf && $oldValue && $oldValue !==
$html ) {
+ $cache->delete( $cacheKey ); // update
all datacenters
+ $ttl = $cache::TTL_UNCACHEABLE; //
don't bother due to delete() tombstone
+ } else {
+ $virtualMtime = ( $oldValue === $html )
? $oldAsOf : false;
+ $ttl = $cache->adaptiveTTL(
$virtualMtime, $ttl, self::MIN_CACHE_TTL );
+ }
+
+ return $html;
+ },
+ [ 'minAsOf' => $safeAsOf ]
);
}
protected function renderWikidataAnnotation( $wdMatches ) {
$entityId = $wdMatches[2];
$currentLang = $this->getLanguage()->getCode();
+ $safeAsOf = $this->getSafeCacheAsOfForUser( 'wikidatawiki' );
$cache = ObjectCache::getMainWANInstance();
+ $cacheKey = $cache->makeKey( 'fileannotations',
'wikidataentity', $currentLang, $entityId );
return $cache->getWithSetCallback(
- $cache->makeKey( 'fileannotations', 'wikidataentity',
$currentLang, $entityId ),
- self::CACHE_TTL,
- function ( $oldValue, &$ttl, array &$setOpts ) use (
$entityId, $currentLang ) {
+ $cacheKey,
+ self::MAX_CACHE_TTL,
+ function ( $oldValue, &$ttl, array &$setOpts, $oldAsOf )
+ use ( $cache, $entityId, $currentLang, $safeAsOf,
$cacheKey ) {
$client = new MultiHttpClient( [] );
$response = $client->run( [
@@ -218,9 +258,14 @@
],
] );
- $entityApiData = json_decode(
$response['body'], true );
+ if ( $response['code'] == 200 ) {
+ $entityApiData = json_decode(
$response['body'], true );
+ $entity =
$entityApiData['entities'][$entityId];
+ } else {
+ $ttl = $cache::TTL_UNCACHEABLE;
- $entity = $entityApiData['entities'][$entityId];
+ return '';
+ }
$labels = $entity['labels'];
$descriptions = $entity['descriptions'];
@@ -289,8 +334,17 @@
}
}
+ if ( $safeAsOf && $oldValue && $oldValue !==
$parsed ) {
+ $cache->delete( $cacheKey ); // update
all datacenters
+ $ttl = $cache::TTL_UNCACHEABLE; //
don't bother due to delete() tombstone
+ } else {
+ $virtualMtime = ( $oldValue === $parsed
) ? $oldAsOf : false;
+ $ttl = $cache->adaptiveTTL(
$virtualMtime, $ttl, self::MIN_CACHE_TTL );
+ }
+
return $parsed;
- }
+ },
+ [ 'minAsOf' => $safeAsOf ]
);
}
@@ -422,4 +476,31 @@
],
];
}
+
+ /**
+ * If a user recently made changes to one of the shared wiki's, try to
avoid using a
+ * stale cache keys for the fileinfo API queries to that wiki and also
purge the keys
+ * if they are outdated, so that it shows in all datacenters
+ *
+ * @param string $dbName
+ * @return mixed
+ */
+ protected function getSafeCacheAsOfForUser( $dbName ) {
+ // If this site is part of the WMF cluster, these timestamp
will be set
+ $lbFactory =
MediaWikiServices::getInstance()->getDBLoadBalancerFactory();
+ $touched = $lbFactory->getChronologyProtectorTouched( $dbName );
+ if ( $touched == false ) {
+ return false;
+ }
+
+ if ( ( microtime( true ) - $touched ) < 3 ) {
+ // Do not slow down the post-save redirect, which
results in a page view.
+ // Wait a few seconds so we can be sure the client has
already followed
+ // the redirect, starting the backend request to view
the page.
+ return false;
+ }
+
+ // Account for DB replica lag with HOLDOFF_TTL
+ return ( $touched + WANObjectCache::HOLDOFF_TTL );
+ }
}
--
To view, visit https://gerrit.wikimedia.org/r/309412
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Id71bef631888ba3b12b9db59bd62bc2fe3647ea8
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/FileAnnotations
Gerrit-Branch: master
Gerrit-Owner: Aaron Schulz <asch...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
