Dan-nl has uploaded a new change for review.
https://gerrit.wikimedia.org/r/96353
Change subject: job-queue-size
......................................................................
job-queue-size
aaron sschulz requested that we use the
JobQueueGroup::singleton()->getQueueSizes() in addition
to the Config::$job_throttle to avoid flooding a queue
Change-Id: I5ef09adb92d2bd6f790a8fab9c62aeb65450cb54
---
M includes/Config.php
M includes/Handlers/Forms/MetadataMappingHandler.php
M includes/Handlers/UploadHandler.php
M includes/Handlers/Xml/XmlMappingHandler.php
M includes/Jobs/UploadMetadataJob.php
5 files changed, 96 insertions(+), 15 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/GWToolset
refs/changes/53/96353/1
diff --git a/includes/Config.php b/includes/Config.php
index b40e8eb..b47fbd5 100644
--- a/includes/Config.php
+++ b/includes/Config.php
@@ -63,11 +63,6 @@
/**
* @var {int}
- */
- public static $job_throttle = 10;
-
- /**
- * @var {int}
* 1.25e7 or 12,500,000 default
*/
public static $max_image_area = 6.4e7;
@@ -79,6 +74,21 @@
* the wiki’s $wgMaxUploadSize is used
*/
public static $max_upload_filesize = 0;
+
+ /**
+ * @var {int}
+ * the maximum number of mediafile jobs to add to the job queue during
this run.
+ */
+ public static $mediafile_job_throttle = 10;
+
+ /**
+ * @var {int}
+ * the maximum number of UploadMediafileJob’s that should exist in the
overall job queue.
+ * if this number is reached, the UploadMetadataJob will attempt
+ * Config::$metadata_job_max_attempts times to add to the additional
+ * UploadMediafileJob’s before failing.
+ */
+ public static $mediafile_job_queue_max = 1000;
/**
* @var {array}
@@ -112,6 +122,15 @@
public static $metadata_file_category = 'GWToolset_Metadata_Sets';
/**
+ * @var {int}
+ * the maximum number of times the UploadMetadataJob will attempt to
add the same
+ * UploadMediafileJob’s to the job queue. this max is used when the
+ * Config::$metadata_job_max_attempts has been reached and if used, can
indicate
+ * an issue with the job queue clearing out UploadMediafileJob’s.
+ */
+ public static $metadata_job_max_attempts = 5;
+
+ /**
* @var {string}
* wiki namespace to store metadata mappings and data sets
*/
diff --git a/includes/Handlers/Forms/MetadataMappingHandler.php
b/includes/Handlers/Forms/MetadataMappingHandler.php
index f69e21f..7262acf 100644
--- a/includes/Handlers/Forms/MetadataMappingHandler.php
+++ b/includes/Handlers/Forms/MetadataMappingHandler.php
@@ -107,6 +107,7 @@
NS_USER
),
array(
+ 'attempts' => 1,
'user-name' => $this->User->getName(),
'whitelisted-post' => $this->_whitelisted_post
)
@@ -408,7 +409,7 @@
}
// at this point
- // * the UploadMetadataJob has created (
Config::$job_throttle ) number of
+ // * the UploadMetadataJob has created (
Config::$mediafile_job_throttle ) number of
// UploadMediafileJobs
// * $user_options['gwtoolset-record-begin'] is the
value that the UploadMetadataJob
// began with
@@ -416,8 +417,8 @@
// processed
//
// example to illustrate the test
- // * Config::$preview_throttle = 3
- // * Config::$job_throttle = 10
+ // * Config::$preview_throttle = 3
+ // * Config::$mediafile_job_throttle = 10
// * $user_options['gwtoolset-record-count'] = 14
// * $user_options['gwtoolset-record-begin'] = 4 (
because the preview took care of 3 )
// * $user_options['gwtoolset-record-current'] = 14 (
13 mediafiles will have been
@@ -429,7 +430,7 @@
// * create another UploadMetadataJob that will take
care of the last record
if (
(int)$user_options['gwtoolset-record-count']
- >= (
(int)$user_options['gwtoolset-record-begin'] + (int)Config::$job_throttle )
+ >= (
(int)$user_options['gwtoolset-record-begin'] +
(int)Config::$mediafile_job_throttle )
) {
$this->_whitelisted_post['gwtoolset-record-begin'] =
(int)$user_options['gwtoolset-record-current'];
@@ -492,7 +493,7 @@
);
if ( $user_options['preview'] === true ) {
- Config::$job_throttle = Config::$preview_throttle;
+ (int)Config::$mediafile_job_throttle =
(int)Config::$preview_throttle;
$mediafile_titles = $this->processMetadata(
$user_options );
$result = PreviewForm::getForm(
diff --git a/includes/Handlers/UploadHandler.php
b/includes/Handlers/UploadHandler.php
index 1f1ca14..a74059d 100644
--- a/includes/Handlers/UploadHandler.php
+++ b/includes/Handlers/UploadHandler.php
@@ -547,7 +547,7 @@
) {
$result = false;
- if ( count( $this->mediafile_jobs ) > Config::$job_throttle ) {
+ if ( count( $this->mediafile_jobs ) >
(int)Config::$mediafile_job_throttle ) {
throw new MWException(
wfMessage( 'gwtoolset-developer-issue' )
->params(
diff --git a/includes/Handlers/Xml/XmlMappingHandler.php
b/includes/Handlers/Xml/XmlMappingHandler.php
index da859f4..5e52cb6 100644
--- a/includes/Handlers/Xml/XmlMappingHandler.php
+++ b/includes/Handlers/Xml/XmlMappingHandler.php
@@ -340,8 +340,8 @@
// stop processing if the current
record nr is >=
// the record nr we should begin
processing on plus the job throttle
if (
-
$user_options['gwtoolset-record-current']
- >=
$user_options['gwtoolset-record-begin'] + Config::$job_throttle
+
(int)$user_options['gwtoolset-record-current']
+ >=
(int)$user_options['gwtoolset-record-begin'] +
(int)Config::$mediafile_job_throttle
) {
$result['stop-reading'] = true;
break;
diff --git a/includes/Jobs/UploadMetadataJob.php
b/includes/Jobs/UploadMetadataJob.php
index f85b6ed..1681051 100644
--- a/includes/Jobs/UploadMetadataJob.php
+++ b/includes/Jobs/UploadMetadataJob.php
@@ -9,8 +9,13 @@
namespace GWToolset\Jobs;
use Job,
+ JobQueueGroup,
+ GWToolset\Config,
+ GWToolset\Constants,
GWToolset\GWTException,
GWToolset\Handlers\Forms\MetadataMappingHandler,
+ MWException,
+ Title,
User;
/**
@@ -55,14 +60,65 @@
}
/**
+ * @return {bool}
+ */
+ protected function recreateMetadataJob() {
+ $result = false;
+
+ if ( (int)$this->params['attempts'] >
(int)Config::$metadata_job_max_attempts ) {
+ throw new MWException(
+ 'There is a serious problem with the
gwtoolsetUploadMediafileJob job queue' .
+ 'There are > ' .
Config::$mediafile_job_queue_max . ' gwtoolsetUploadMediafileJob’s ' .
+ 'in the job queue. gwtoolsetUploadMetadataJob
has attempted ' .
+ Config::$metadata_job_max_attempts . ' times to
add additional ' .
+ 'gwtoolsetUploadMediafileJob’s to the job
queue, but cannot because the ' .
+ 'gwtoolsetUploadMediafileJob’s are not clearing
out.'
+ );
+ }
+
+ $job = new UploadMetadataJob(
+ Title::newFromText(
+ User::newFromName( $this->params['user-name'] )
. '/' .
+ Constants::EXTENSION_NAME . '/' .
+ 'Metadata Batch Job/' .
+ uniqid(),
+ NS_USER
+ ),
+ array(
+ 'attempts' => (int)$this->params['attempts'] +
1,
+ 'user-name' => $this->params['user-name'],
+ 'whitelisted-post' =>
$this->params['whitelisted-post']
+ )
+ );
+
+ $result = JobQueueGroup::singleton()->push( $job );
+ }
+
+ /**
* entry point
- * @todo: should $result always be true?
- * @return {bool|array}
+ * @return {bool}
*/
public function run() {
$result = false;
if ( !$this->validateParams() ) {
+ return $result;
+ }
+
+ $job_queue_sizes = JobQueueGroup::singleton()->getQueueSizes();
+
+ if ( isset( $job_queue_sizes['gwtoolsetUploadMediafileJob'] )
+ && $job_queue_sizes['gwtoolsetUploadMediafileJob'] >
Config::$mediafile_job_queue_max
+ ) {
+ $result = $this->recreateMetadataJob();
+
+ if ( !$result ) {
+ $this->setLastError(
+ __METHOD__ . ': ' .
+ wfMessage(
'gwtoolset-batchjob-metadata-creation-failure' )->escaped()
+ );
+ }
+
return $result;
}
@@ -81,6 +137,11 @@
protected function validateParams() {
$result = true;
+ if ( empty( $this->params['attempts'] ) ) {
+ $this->setLastError( __METHOD__ . ': no
$this->params[\'attempts\'] provided' );
+ $result = false;
+ }
+
if ( empty( $this->params['user-name'] ) ) {
$this->setLastError( __METHOD__ . ': no
$this->params[\'user-name\'] provided' );
$result = false;
--
To view, visit https://gerrit.wikimedia.org/r/96353
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I5ef09adb92d2bd6f790a8fab9c62aeb65450cb54
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/GWToolset
Gerrit-Branch: master
Gerrit-Owner: Dan-nl <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
