Aaron Schulz has uploaded a new change for review.
https://gerrit.wikimedia.org/r/252895
Change subject: [WIP] Add $wgCdnReboundPurgeDelay for more consistent CDN purges
......................................................................
[WIP] Add $wgCdnReboundPurgeDelay for more consistent CDN purges
Bug: T113192
Change-Id: I89deb4f8143c1cc6154cdd05bcee1f49d3e3a75a
---
M RELEASE-NOTES-1.27
M autoload.php
M includes/DefaultSettings.php
M includes/deferred/SquidUpdate.php
A includes/jobqueue/jobs/CdnPurgeJob.php
5 files changed, 78 insertions(+), 4 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/95/252895/1
diff --git a/RELEASE-NOTES-1.27 b/RELEASE-NOTES-1.27
index e124832..73c65ef 100644
--- a/RELEASE-NOTES-1.27
+++ b/RELEASE-NOTES-1.27
@@ -78,6 +78,8 @@
and .flex(@grow: 1, @shrink: 1, @width: auto, @order: 1)) in Less to create
cross-browser-compatible FlexBox rules. Users will still need to add fallback
float rules or the like for compatibility with IE9- separately.
+* $wgCdnReboundPurgeDelay was added to provide secondary delayed purges of URLs
+ from CDN to mitigate DB replication lag and WAN cache purge lag.
==== External libraries ====
diff --git a/autoload.php b/autoload.php
index 15f1b10..5d116ed 100644
--- a/autoload.php
+++ b/autoload.php
@@ -201,6 +201,7 @@
'CdbException' => __DIR__ . '/includes/compat/CdbCompat.php',
'CdbReader' => __DIR__ . '/includes/compat/CdbCompat.php',
'CdbWriter' => __DIR__ . '/includes/compat/CdbCompat.php',
+ 'CdnPurgeJob' => __DIR__ . '/includes/jobqueue/jobs/CdnPurgeJob.php',
'CgzCopyTransaction' => __DIR__ .
'/maintenance/storage/recompressTracked.php',
'ChangePassword' => __DIR__ . '/maintenance/changePassword.php',
'ChangeTags' => __DIR__ . '/includes/changetags/ChangeTags.php',
diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php
index bf6e245..b786737 100644
--- a/includes/DefaultSettings.php
+++ b/includes/DefaultSettings.php
@@ -2596,6 +2596,15 @@
$wgCdnMaxageLagged = 30;
/**
+ * If set, any SquidPurge call on a URL or URLs will send a second purge no
less than
+ * this many seconds later via the job queue. This requires delayed job
support.
+ * This should be safely higher than the 'max lag' value in $wgLBFactoryConf.
+ *
+ * @since 1.27
+ */
+$wgCdnReboundPurgeDelay = 0;
+
+/**
* Default maximum age for raw CSS/JS accesses
*
* 300 seconds = 5 minutes.
@@ -6744,6 +6753,7 @@
'refreshLinksPrioritized' => 'RefreshLinksJob', // for cascading
protection
'refreshLinksDynamic' => 'RefreshLinksJob', // for pages with dynamic
content
'activityUpdateJob' => 'ActivityUpdateJob',
+ 'cdnPurge' => 'CdnPurgeJob',
'enqueue' => 'EnqueueJob', // local queue for multi-DC setups
'null' => 'NullJob'
);
diff --git a/includes/deferred/SquidUpdate.php
b/includes/deferred/SquidUpdate.php
index 0e4161c..99ab2e5 100644
--- a/includes/deferred/SquidUpdate.php
+++ b/includes/deferred/SquidUpdate.php
@@ -29,6 +29,8 @@
/** @var string[] Collection of URLs to purge */
protected $urls = array();
+ const PURGE_NO_REBOUND = 1; // do NOT enqueue a rebound purge
+
/**
* @param array $urlArr Collection of URLs to purge
*/
@@ -66,9 +68,11 @@
/**
* Purges the list of URLs passed to the constructor.
+ *
+ * @param integer $flags Bitfield of SquidUpdate::PURGE_* constants
*/
- public function doUpdate() {
- self::purge( $this->urls );
+ public function doUpdate( $flags = 0 ) {
+ self::purge( $this->urls, $flags );
}
/**
@@ -78,9 +82,10 @@
* XXX report broken Squids per mail or log
*
* @param array $urlArr List of full URLs to purge
+ * @param integer $flags Bitfield of SquidUpdate::PURGE_* constants
*/
- public static function purge( array $urlArr ) {
- global $wgSquidServers, $wgHTCPRouting;
+ public static function purge( array $urlArr, $flags = 0 ) {
+ global $wgSquidServers, $wgHTCPRouting, $wgCdnReboundPurgeDelay;
if ( !$urlArr ) {
return;
@@ -119,6 +124,16 @@
$pool->run();
}
+
+ if ( $wgCdnReboundPurgeDelay > 0 && !( $flags &
self::PURGE_NO_REBOUND ) ) {
+ JobQueueGroup::singleton()->lazyPush( new CdnPurgeJob(
+ Title::makeTitle( NS_SPECIAL, 'Badtitle/' .
__CLASS__ ),
+ array(
+ 'urls' => $urlArr,
+ 'jobReleaseTimestamp' => time() +
$wgCdnReboundPurgeDelay
+ )
+ ) );
+ }
}
/**
diff --git a/includes/jobqueue/jobs/CdnPurgeJob.php
b/includes/jobqueue/jobs/CdnPurgeJob.php
new file mode 100644
index 0000000..0f1ebb9
--- /dev/null
+++ b/includes/jobqueue/jobs/CdnPurgeJob.php
@@ -0,0 +1,46 @@
+<?php
+/**
+ * Job to purge a set of URLs from CDN
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup JobQueue
+ */
+
+/**
+ * Job to purge a set of URLs from CDN
+ *
+ * @ingroup JobQueue
+ * @since 1.27
+ */
+class CdnPurgeJob extends Job {
+ /**
+ * @param Title $title
+ * @param array $params Job parameters (urls)
+ */
+ function __construct( Title $title, array $params ) {
+ parent::__construct( 'cdnPurge', $title, $params );
+ $this->removeDuplicates = false; // delay semantics are critical
+ }
+
+ public function run() {
+ $update = new SquidUpdate( $this->params['urls'] );
+ $update->doUpdate( SquidUpdate::PURGE_NO_REBOUND ); // avoid
recursion
+
+ return true;
+ }
+}
--
To view, visit https://gerrit.wikimedia.org/r/252895
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I89deb4f8143c1cc6154cdd05bcee1f49d3e3a75a
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Aaron Schulz <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
