Aaron Schulz has uploaded a new change for review.
https://gerrit.wikimedia.org/r/90726
Change subject: Added "maxPartitionsTry" option to JobQueueFederated
......................................................................
Added "maxPartitionsTry" option to JobQueueFederated
* Also throw an error if some jobs are not inserted to match the
other queues.
Change-Id: I363e4813ee89468cde4349ba81ea458ff4ffe446
---
M includes/job/JobQueueFederated.php
1 file changed, 20 insertions(+), 10 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/26/90726/1
diff --git a/includes/job/JobQueueFederated.php
b/includes/job/JobQueueFederated.php
index d3ce164..36f4959 100644
--- a/includes/job/JobQueueFederated.php
+++ b/includes/job/JobQueueFederated.php
@@ -56,6 +56,8 @@
/** @var BagOStuff */
protected $cache;
+ protected $maxPartitionsTry; // integer; maximum number of partitions
to try
+
const CACHE_TTL_SHORT = 30; // integer; seconds to cache info without
re-validating
const CACHE_TTL_LONG = 300; // integer; seconds to cache info that is
kept up to date
@@ -72,6 +74,10 @@
* the federated queue itself (e.g. 'order'
and 'claimTTL').
* - partitionsNoPush : List of partition names that can handle
pop() but not push().
* This can be used to migrate away from a
certain partition.
+ * - maxPartitionsTry : Maximum number of times to attempt job
insertion using
+ * different partition queues. This improves
availability
+ * during failure, at the cost of added
latency and somewhat
+ * less reliable job de-duplication mechanisms.
* @param array $params
*/
protected function __construct( array $params ) {
@@ -82,6 +88,9 @@
if ( !isset( $params['partitionsBySection'][$section] ) ) {
throw new MWException( "No configuration for section
'$section'." );
}
+ $this->maxPartitionsTry = isset( $params['maxPartitionsTry'] )
+ ? $params['maxPartitionsTry']
+ : 2;
// Get the full partition map
$this->partitionMap = $params['partitionsBySection'][$section];
arsort( $this->partitionMap, SORT_NUMERIC );
@@ -94,10 +103,10 @@
}
// Get the config to pass to merge into each partition queue
config
$baseConfig = $params;
- foreach ( array( 'class', 'sectionsByWiki',
+ foreach ( array( 'class', 'sectionsByWiki', 'maxPartitionsTry',
'partitionsBySection', 'configByPartition',
'partitionsNoPush' ) as $o )
{
- unset( $baseConfig[$o] );
+ unset( $baseConfig[$o] ); // partition queue doesn't
care about this
}
// Get the partition queue objects
foreach ( $this->partitionMap as $partition => $w ) {
@@ -194,16 +203,17 @@
}
protected function doBatchPush( array $jobs, $flags ) {
- if ( !count( $jobs ) ) {
- return true; // nothing to do
- }
// Local ring variable that may be changed to point to a new
ring on failure
$partitionRing = $this->partitionPushRing;
- // Try to insert the jobs and update $partitionsTry on any
failures
- $jobsLeft = $this->tryJobInsertions( $jobs, $partitionRing,
$flags );
- if ( count( $jobsLeft ) ) { // some jobs failed to insert?
- // Try to insert the remaning jobs once more, ignoring
the bad partitions
- return !count( $this->tryJobInsertions( $jobsLeft,
$partitionRing, $flags ) );
+ // Try to insert the jobs and update $partitionsTry on any
failures.
+ // Retry to insert any remaning jobs again, ignoring the bad
partitions.
+ $jobsLeft = $jobs;
+ for ( $i = $this->maxPartitionsTry; $i > 0 && count( $jobsLeft
); --$i ) {
+ $jobsLeft = $this->tryJobInsertions( $jobsLeft,
$partitionRing, $flags );
+ }
+ if ( count( $jobsLeft ) ) {
+ throw new JobQueueError(
+ "Could not insert job(s),
{$this->maxPartitionsTry} partitions tried." );
}
return true;
}
--
To view, visit https://gerrit.wikimedia.org/r/90726
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I363e4813ee89468cde4349ba81ea458ff4ffe446
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Aaron Schulz <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
