Aaron Schulz has uploaded a new change for review.
https://gerrit.wikimedia.org/r/53306
Change subject: [JobQueue] Try to cut down on waitForBackups() calls in
runJobs.php.
......................................................................
[JobQueue] Try to cut down on waitForBackups() calls in runJobs.php.
Change-Id: I2fc97ef8dbc02d1184959ba962dcafdae9fae808
---
M includes/job/JobQueueGroup.php
M maintenance/runJobs.php
2 files changed, 26 insertions(+), 4 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/06/53306/1
diff --git a/includes/job/JobQueueGroup.php b/includes/job/JobQueueGroup.php
index 50b5471..351c71a 100644
--- a/includes/job/JobQueueGroup.php
+++ b/includes/job/JobQueueGroup.php
@@ -202,6 +202,25 @@
}
/**
+ * Wait for any slaves or backup queue servers to catch up.
+ *
+ * This does nothing for certain queue classes.
+ *
+ * @return void
+ * @throws MWException
+ */
+ public function waitForBackups() {
+ global $wgJobTypeConf;
+
+ wfProfileIn( __METHOD__ );
+ // Try to avoid doing this more than once per queue storage
medium
+ foreach ( $wgJobTypeConf as $type => $conf ) {
+ $this->get( $type )->waitForBackups();
+ }
+ wfProfileOut( __METHOD__ );
+ }
+
+ /**
* Get the list of queue types
*
* @return array List of strings
diff --git a/maintenance/runJobs.php b/maintenance/runJobs.php
index 714bb84..b1be714 100644
--- a/maintenance/runJobs.php
+++ b/maintenance/runJobs.php
@@ -73,7 +73,7 @@
$type = $this->getOption( 'type', false );
$wgTitle = Title::newFromText( 'RunJobs.php' );
$dbw = wfGetDB( DB_MASTER );
- $n = 0;
+ $jobsRun = 0; // counter
$group = JobQueueGroup::singleton();
// Handle any required periodic queue maintenance
@@ -88,6 +88,7 @@
? $group->pop( JobQueueGroup::TYPE_DEFAULT,
JobQueueGroup::USE_CACHE )
: $group->pop( $type ); // job from a single
queue
if ( $job ) { // found a job
+ ++$jobsRun;
$this->runJobsLog( $job->toString() . "
STARTING" );
// Run the job...
@@ -113,19 +114,21 @@
}
// Break out if we hit the job count or wall
time limits...
- if ( $maxJobs && ++$n >= $maxJobs ) {
+ if ( $maxJobs && $jobsRun >= $maxJobs ) {
break;
} elseif ( $maxTime && ( time() - $startTime )
> $maxTime ) {
break;
}
- // Don't let any queue slaves/backups fall
behind
- $group->get( $job->getType()
)->waitForBackups();
// Don't let any of the main DB slaves get
backed up
$timePassed = time() - $lastTime;
if ( $timePassed >= 5 || $timePassed < 0 ) {
wfWaitForSlaves();
}
+ // Don't let any queue slaves/backups fall
behind
+ if ( $jobsRun > 0 && ( $jobsRun % 100 ) == 0 ) {
+ $group->waitForBackups();
+ }
}
} while ( $job ); // stop when there are no jobs
}
--
To view, visit https://gerrit.wikimedia.org/r/53306
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I2fc97ef8dbc02d1184959ba962dcafdae9fae808
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Aaron Schulz <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
