jenkins-bot has submitted this change and it was merged.
Change subject: Fix a typo in BC code that handles toId => toPageId
......................................................................
Fix a typo in BC code that handles toId => toPageId
There was a typo that could explain why CheckerJobs are checking a
huge amount pages. $toId was certainly set to 0 causing range( $from,
$id ) to generate a huge list of chunks.
Added a bunch of new checks to make sure that currupted jobs will be
ignored.
Bug: T143862
Change-Id: I97735a28b8f64339104d7ed843cd222a1e03f945
---
M includes/Job/CheckerJob.php
1 file changed, 58 insertions(+), 6 deletions(-)
Approvals:
Cindy-the-browser-test-bot: Looks good to me, but someone else must approve
EBernhardson: Looks good to me, approved
jenkins-bot: Verified
diff --git a/includes/Job/CheckerJob.php b/includes/Job/CheckerJob.php
index b067855..db7682b 100644
--- a/includes/Job/CheckerJob.php
+++ b/includes/Job/CheckerJob.php
@@ -32,6 +32,12 @@
*/
class CheckerJob extends Job {
/**
+ * @const int max number of retries, 3 means that the job can be run at
+ * most 4 times.
+ */
+ const JOB_MAX_RETRIES = 3;
+
+ /**
* Construct a new CherckerJob.
* @param int $fromPageId
* @param int $toPageId
@@ -63,7 +69,7 @@
$params['fromPageId'] = $params['fromId'];
unset( $params['fromId'] );
}
- if ( isset( $parmas['toId'] ) ) {
+ if ( isset( $params['toId'] ) ) {
$params['toPageId'] = $params['toId'];
unset( $params['toId'] );
}
@@ -100,6 +106,14 @@
return false;
}
+ $chunkSize = isset( $profile['jobs_chunk_size'] ) ?
$profile['jobs_chunk_size'] : null;
+ if ( !$chunkSize || $chunkSize < 0 ) {
+ LoggerFactory::getInstance( 'CirrusSearch' )->warning(
+ "Cannot run CheckerJob invalid jobs_chunk_size,
check CirrusSearchSanityCheck config."
+ );
+ return false;
+ }
+
$maxTime = isset( $profile['checker_job_max_time'] ) ?
$profile['checker_job_max_time'] : null;
if ( !$maxTime || $maxTime < 0 ) {
LoggerFactory::getInstance( 'CirrusSearch' )->warning(
@@ -108,10 +122,35 @@
return false;
}
- $startTime = time();
+ $from = $this->params['fromPageId'];
+ $to = $this->params['toPageId'];
+
+ if ( $from > $to ) {
+ LoggerFactory::getInstance( 'CirrusSearch' )->warning(
+ "Cannot run CheckerJob: from > to ( {from} >
{to} ), job is corrupted?",
+ [
+ 'from' => $from,
+ 'to' => $to,
+ ]
+ );
+ return false;
+ }
+
+ if ( ( $to - $from ) > $chunkSize ) {
+ LoggerFactory::getInstance( 'CirrusSearch' )->warning(
+ "Cannot run CheckerJob: to - from > chunkSize(
{from}, {to} > {chunkSize} ), job is corrupted or profile mismatch?",
+ [
+ 'from' => $from,
+ 'to' => $to,
+ 'chunkSize' => $chunkSize,
+ ]
+ );
+ return false;
+ }
$connections = $this->decideClusters();
$clusterNames = implode( ', ', array_keys( $connections ) );
+
LoggerFactory::getInstance( 'CirrusSearch' )->debug(
"Running CheckerJob on cluster $clusterNames {diff}s
after insertion",
[
@@ -120,8 +159,7 @@
]
);
- $from = $this->params['fromPageId'];
- $to = $this->params['toPageId'];
+ $startTime = time();
$pageCache = new ArrayObject();
$checkers = [];
@@ -191,15 +229,29 @@
* @param int $newFrom the new from offset
*/
private function retry( $cause, $newFrom ) {
+ if ( $this->params['retryCount'] >= self::JOB_MAX_RETRIES ) {
+ LoggerFactory::getInstance( 'CirrusSearch' )->info(
+ "Sanitize CheckerJob: $cause
({fromPageId}:{toPageId}), Abandonning CheckerJob after {retries} retries,
(jobs_chunk_size too high?).",
+ [
+ 'retries' =>
$this->params['retryCount'],
+ 'fromPageId' =>
$this->params['fromPageId'],
+ 'toPageId' => $this->params['toPageId'],
+ ]
+ );
+ return;
+ }
+
$delay = self::backoffDelay( $this->params['retryCount'] );
$job = clone $this;
$job->params['retryCount']++;
$job->params['fromPageId'] = $newFrom;
$job->setDelay( $delay );
LoggerFactory::getInstance( 'CirrusSearch' )->info(
- "Sanitize CheckerJob: $cause, Requeueing CheckerJob
with a delay of {delay}s.",
+ "Sanitize CheckerJob: $cause ({fromPageId}:{toPageId}),
Requeueing CheckerJob with a delay of {delay}s.",
[
- 'delay' => $delay
+ 'delay' => $delay,
+ 'fromPageId' => $job->params['fromPageId'],
+ 'toPageId' => $job->params['toPageId'],
]
);
JobQueueGroup::singleton()->push( $job );
--
To view, visit https://gerrit.wikimedia.org/r/306649
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I97735a28b8f64339104d7ed843cd222a1e03f945
Gerrit-PatchSet: 5
Gerrit-Project: mediawiki/extensions/CirrusSearch
Gerrit-Branch: master
Gerrit-Owner: DCausse <[email protected]>
Gerrit-Reviewer: Cindy-the-browser-test-bot <[email protected]>
Gerrit-Reviewer: EBernhardson <[email protected]>
Gerrit-Reviewer: Gehel <[email protected]>
Gerrit-Reviewer: Manybubbles <[email protected]>
Gerrit-Reviewer: Smalyshev <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
