jenkins-bot has submitted this change and it was merged.
Change subject: Bug 51156: Don't subclass HTMLCacheUpdate any more
......................................................................
Bug 51156: Don't subclass HTMLCacheUpdate any more
This fixes template and file updates for templates / files that are used in
more than 10 pages. Tested locally with templates used 20+ times.
TODO: Subclass from generic range splitting base class once that is available
instead of duplicating the range splitting functionality from HTMLCacheUpdate.
Change-Id: I4d74b4b007add0758f726c3311367f9853868aeb
---
M php/Parsoid.php
M php/ParsoidCacheUpdateJob.php
2 files changed, 154 insertions(+), 5 deletions(-)
Approvals:
Aaron Schulz: Looks good to me, approved
jenkins-bot: Verified
diff --git a/php/Parsoid.php b/php/Parsoid.php
index 37301f5..829c3cd 100644
--- a/php/Parsoid.php
+++ b/php/Parsoid.php
@@ -64,7 +64,8 @@
* @return void
*/
protected static function setupDefaultConfig() {
- global $wgParsoidCacheServers, $wgParsoidSkipRatio, $wgDBname;
+ global $wgParsoidCacheServers, $wgParsoidSkipRatio, $wgDBname,
+ $wgParsoidMaxBacklinksInvalidate;
/**
* An array of Varnish caches in front of Parsoid to keep up to
date.
@@ -83,6 +84,11 @@
$wgParsoidWikiPrefix = $wgDBname;
/**
+ * The maximum number of backlinks (templates and files) to
update.
+ */
+ $wgParsoidMaxBacklinksInvalidate = false;
+
+ /**
* The portion of update requests to skip for basic load
shedding. A
* float between 0 (none are skipped) and 1 (all are skipped).
*/
diff --git a/php/ParsoidCacheUpdateJob.php b/php/ParsoidCacheUpdateJob.php
index 9367710..3773417 100644
--- a/php/ParsoidCacheUpdateJob.php
+++ b/php/ParsoidCacheUpdateJob.php
@@ -4,9 +4,14 @@
* HTML cache refreshing and -invalidation job for the Parsoid varnish caches.
* See
*
http://www.mediawiki.org/wiki/Parsoid/Minimal_performance_strategy_for_July_release
- * @TODO: eventually extend some generic backlink job base class in core
+ * @TODO: This is mostly a copy of the HTMLCacheUpdate code. Eventually extend
+ * some generic backlink job base class in core
*/
-class ParsoidCacheUpdateJob extends HTMLCacheUpdateJob {
+class ParsoidCacheUpdateJob extends Job {
+ /** @var BacklinkCache */
+ protected $blCache;
+
+ protected $rowsPerJob;
/**
* Construct a job
@@ -15,13 +20,13 @@
* @param $id Integer: job id
*/
function __construct( $title, $params, $id = 0 ) {
- wfDebug( "ParsoidCacheUpdateJob.__construct\n" );
+ wfDebug( "ParsoidCacheUpdateJob.__construct " . $title . "\n" );
global $wgUpdateRowsPerJob;
Job::__construct( 'ParsoidCacheUpdateJob', $title, $params, $id
);
# $this->rowsPerJob = $wgUpdateRowsPerJob;
- # Parsoid re-parses will be slow, so set the number of titles
per job
+ # Parsoid re-parses can be slow, so set the number of titles
per job
# to 10 for now.
$this->rowsPerJob = 10;
@@ -52,6 +57,144 @@
}
/**
+ * Update all of the backlinks
+ */
+ protected function doFullUpdate() {
+ global $wgParsoidMaxBacklinksInvalidate;
+
+ # Get an estimate of the number of rows from the BacklinkCache
+ $max = max( $this->rowsPerJob * 2,
$wgParsoidMaxBacklinksInvalidate ) + 1;
+ $numRows = $this->blCache->getNumLinks( $this->params['table'],
$max );
+ if ( $wgParsoidMaxBacklinksInvalidate !== false
+ && $numRows > $wgParsoidMaxBacklinksInvalidate ) {
+ wfDebug( "Skipped HTML cache invalidation of
{$this->title->getPrefixedText()}." );
+ return true;
+ }
+
+ if ( $numRows > $this->rowsPerJob * 2 ) {
+ # Do fast cached partition
+ $this->insertPartitionJobs();
+ } else {
+ # Get the links from the DB
+ $titleArray = $this->blCache->getLinks(
$this->params['table'] );
+ # Check if the row count estimate was correct
+ if ( $titleArray->count() > $this->rowsPerJob * 2 ) {
+ # Not correct, do accurate partition
+ wfDebug( __METHOD__ . ": row count estimate was
incorrect, repartitioning\n" );
+ $this->insertJobsFromTitles( $titleArray );
+ } else {
+ $this->invalidateTitles( $titleArray ); // just
do the query
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * Update some of the backlinks, defined by a page ID range
+ */
+ protected function doPartialUpdate() {
+ $titleArray = $this->blCache->getLinks(
+ $this->params['table'], $this->params['start'],
$this->params['end'] );
+ if ( $titleArray->count() <= $this->rowsPerJob * 2 ) {
+ # This partition is small enough, do the update
+ $this->invalidateTitles( $titleArray );
+ } else {
+ # Partitioning was excessively inaccurate. Divide the
job further.
+ # This can occur when a large number of links are added
in a short
+ # period of time, say by updating a heavily-used
template.
+ $this->insertJobsFromTitles( $titleArray );
+ }
+ return true;
+ }
+
+ /**
+ * Partition the current range given by $this->params['start'] and
$this->params['end'],
+ * using a pre-calculated title array which gives the links in that
range.
+ * Queue the resulting jobs.
+ *
+ * @param $titleArray array
+ * @param $rootJobParams array
+ * @return void
+ */
+ protected function insertJobsFromTitles( $titleArray, $rootJobParams =
array() ) {
+ // Carry over any "root job" information
+ $rootJobParams = $this->getRootJobParams();
+ # We make subpartitions in the sense that the start of the
first job
+ # will be the start of the parent partition, and the end of the
last
+ # job will be the end of the parent partition.
+ $jobs = array();
+ $start = $this->params['start']; # start of the current job
+ $numTitles = 0;
+ foreach ( $titleArray as $title ) {
+ $id = $title->getArticleID();
+ # $numTitles is now the number of titles in the current
job not
+ # including the current ID
+ if ( $numTitles >= $this->rowsPerJob ) {
+ # Add a job up to but not including the current
ID
+ $jobs[] = new ParsoidCacheUpdateJob(
$this->title,
+ array(
+ 'table' =>
$this->params['table'],
+ 'start' => $start,
+ 'end' => $id - 1
+ ) + $rootJobParams // carry over
information for de-duplication
+ );
+ $start = $id;
+ $numTitles = 0;
+ }
+ $numTitles++;
+ }
+ # Last job
+ $jobs[] = new ParsoidCacheUpdateJob( $this->title,
+ array(
+ 'table' => $this->params['table'],
+ 'start' => $start,
+ 'end' => $this->params['end']
+ ) + $rootJobParams // carry over information for
de-duplication
+ );
+ wfDebug( __METHOD__ . ": repartitioning into " . count( $jobs )
. " jobs\n" );
+
+ if ( count( $jobs ) < 2 ) {
+ # I don't think this is possible at present, but
handling this case
+ # makes the code a bit more robust against future code
updates and
+ # avoids a potential infinite loop of repartitioning
+ wfDebug( __METHOD__ . ": repartitioning failed!\n" );
+ $this->invalidateTitles( $titleArray );
+ } else {
+ JobQueueGroup::singleton()->push( $jobs );
+ }
+ }
+
+
+ /**
+ * @param $rootJobParams array
+ * @return void
+ */
+ protected function insertPartitionJobs( $rootJobParams = array() ) {
+ // Carry over any "root job" information
+ $rootJobParams = $this->getRootJobParams();
+
+ $batches = $this->blCache->partition( $this->params['table'],
$this->rowsPerJob );
+ if ( !count( $batches ) ) {
+ return; // no jobs to insert
+ }
+
+ $jobs = array();
+ foreach ( $batches as $batch ) {
+ list( $start, $end ) = $batch;
+ $jobs[] = new ParsoidCacheUpdateJob( $this->title,
+ array(
+ 'table' => $this->params['table'],
+ 'start' => $start,
+ 'end' => $end,
+ ) + $rootJobParams // carry over information
for de-duplication
+ );
+ }
+
+ JobQueueGroup::singleton()->push( $jobs );
+ }
+
+ /**
* Construct a cache server URL
*
* @param $title Title
--
To view, visit https://gerrit.wikimedia.org/r/73368
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I4d74b4b007add0758f726c3311367f9853868aeb
Gerrit-PatchSet: 4
Gerrit-Project: mediawiki/extensions/Parsoid
Gerrit-Branch: master
Gerrit-Owner: GWicke <[email protected]>
Gerrit-Reviewer: Aaron Schulz <[email protected]>
Gerrit-Reviewer: GWicke <[email protected]>
Gerrit-Reviewer: Subramanya Sastry <[email protected]>
Gerrit-Reviewer: Tim Starling <[email protected]>
Gerrit-Reviewer: jenkins-bot
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
