EBernhardson has uploaded a new change for review.
https://gerrit.wikimedia.org/r/249685
Change subject: Support a maxjobs parameter for fast running jobs
......................................................................
Support a maxjobs parameter for fast running jobs
Some jobs, like enqueue and wikibase-addUsagesForPage take less
than 10ms to run. It makes sense to run many of these at once
instead of using rpc/RunJobs.php default of 1 job per invocation.
Change-Id: If41bbaefcee3897525d0851703db8afb050e8463
---
M jobrunner.sample.json
M src/JobRunnerPipeline.php
M src/RedisJobService.php
3 files changed, 12 insertions(+), 2 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/services/jobrunner
refs/changes/85/249685/1
diff --git a/jobrunner.sample.json b/jobrunner.sample.json
index 515f499..086e737 100755
--- a/jobrunner.sample.json
+++ b/jobrunner.sample.json
@@ -99,5 +99,5 @@
"statsd": "statsd.eqiad.wmnet:8125",
// Command used to launch a runner for a given job queue
- "dispatcher": "php /srv/mediawiki/multiversion/MWScript.php runJobs.php
--wiki=%(db)x --type=%(type)x --maxtime=%(maxtime)x --memory-limit=%(maxmem)x
--result=json"
+ "dispatcher": "php /srv/mediawiki/multiversion/MWScript.php runJobs.php
--wiki=%(db)x --type=%(type)x --maxtime=%(maxtime)x --memory-limit=%(maxmem)x
--maxjobs=%(maxjobs)x --result=json"
}
diff --git a/src/JobRunnerPipeline.php b/src/JobRunnerPipeline.php
index a9405f1..f6106f4 100755
--- a/src/JobRunnerPipeline.php
+++ b/src/JobRunnerPipeline.php
@@ -169,11 +169,14 @@
$maxmem = isset( $this->srvc->maxMemMap[$type] )
? $this->srvc->maxMemMap[$type]
: $this->srvc->maxMemMap['*'];
+ $maxjobs = isset( $this->srvc->maxJobsMap[$type] )
+ ? $this->srvc->maxJobsMap[$type]
+ : $this->srvc->maxJobsMap['*'];
// Make sure the runner is launched with various time/memory
limits.
// Nice the process so things like ssh and deployment scripts
are fine.
$what = $with = array();
- foreach ( compact( 'db', 'type', 'maxtime', 'maxmem' ) as $k =>
$v ) {
+ foreach ( compact( 'db', 'type', 'maxtime', 'maxmem',
'maxjobs') as $k => $v ) {
$what[] = "%($k)u";
$with[] = rawurlencode( $v );
$what[] = "%($k)x";
diff --git a/src/RedisJobService.php b/src/RedisJobService.php
index bea9c87..0fe1778 100755
--- a/src/RedisJobService.php
+++ b/src/RedisJobService.php
@@ -28,6 +28,8 @@
public $maxRealMap = array();
/** @var array Map of (job type => integer) */
public $maxMemMap = array();
+ /** @var array Map of (job type => integer) */
+ public $maxJobsMap = array();
/** @var array String command to run jobs and return the status JSON
blob */
public $dispatcher;
@@ -164,6 +166,11 @@
$this->maxMemMap = $config['limits']['memory'] +
$this->maxMemMap;
}
+ $this->maxJobsMap['*'] = '1';
+ if ( isset( $config['limits']['jobs'] ) ) {
+ $this->maxJobsMap = $config['limits']['jobs'] +
$this->maxJobsMap;
+ }
+
if ( isset( $config['statsd'] ) ) {
if ( strpos( $config['statsd'], ':' ) !== false ) {
$parts = explode( ':', $config['statsd'] );
--
To view, visit https://gerrit.wikimedia.org/r/249685
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: If41bbaefcee3897525d0851703db8afb050e8463
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/services/jobrunner
Gerrit-Branch: master
Gerrit-Owner: EBernhardson <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
