Aaron Schulz has uploaded a new change for review.
https://gerrit.wikimedia.org/r/101170
Change subject: Added a page_links_updated column for job de-duplication
......................................................................
Added a page_links_updated column for job de-duplication
Change-Id: I74b6f507ef7371db92e0c3f058d38c0ca5dea9ef
---
M RELEASE-NOTES-1.23
M includes/WikiPage.php
M includes/deferred/LinksUpdate.php
M includes/installer/MysqlUpdater.php
M includes/installer/PostgresUpdater.php
M includes/installer/SqliteUpdater.php
M includes/job/jobs/RefreshLinksJob.php
A maintenance/archives/patch-page_links_updated.sql
M maintenance/postgres/tables.sql
M maintenance/tables.sql
10 files changed, 51 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/core
refs/changes/70/101170/1
diff --git a/RELEASE-NOTES-1.23 b/RELEASE-NOTES-1.23
index 3e77434..f87a913 100644
--- a/RELEASE-NOTES-1.23
+++ b/RELEASE-NOTES-1.23
@@ -108,6 +108,7 @@
* The ExpandTemplates extension has been moved into MediaWiki core.
* (bug 52812) Removed "Disable search suggestions" from Preference.
* (bug 52809) Removed "Disable browser page caching" from Preference.
+* A page_links_updated field has been added to the page table and is required.
== Compatibility ==
diff --git a/includes/WikiPage.php b/includes/WikiPage.php
index 7afc65c..8ea0f5c 100644
--- a/includes/WikiPage.php
+++ b/includes/WikiPage.php
@@ -84,6 +84,11 @@
protected $mTouched = '19700101000000';
/**
+ * @var string
+ */
+ protected $mLinksUpdated = '19700101000000';
+
+ /**
* @var int|null
*/
protected $mCounter = null;
@@ -241,6 +246,7 @@
$this->mRedirectTarget = null; // Title object if set
$this->mLastRevision = null; // Latest revision
$this->mTouched = '19700101000000';
+ $this->mLinksUpdated = '19700101000000';
$this->mTimestamp = '';
$this->mIsRedirect = false;
$this->mLatest = false;
@@ -278,6 +284,7 @@
'page_is_new',
'page_random',
'page_touched',
+ 'page_links_updated',
'page_latest',
'page_len',
);
@@ -405,6 +412,7 @@
$this->mId = intval( $data->page_id );
$this->mCounter = intval( $data->page_counter );
$this->mTouched = wfTimestamp( TS_MW,
$data->page_touched );
+ $this->mLinksUpdated = wfTimestampOrNull( TS_MW,
$data->page_links_updated );
$this->mIsRedirect = intval( $data->page_is_redirect );
$this->mLatest = intval( $data->page_latest );
// Bug 37225: $latest may no longer match the cached
latest Revision object.
@@ -534,6 +542,17 @@
}
/**
+ * Get the page_links_updated field
+ * @return string|null containing GMT timestamp
+ */
+ public function getLinksTimestamp() {
+ if ( !$this->mDataLoaded ) {
+ $this->loadPageData();
+ }
+ return $this->mLinksUpdated;
+ }
+
+ /**
* Get the page_latest field
* @return integer rev_id of current revision
*/
diff --git a/includes/deferred/LinksUpdate.php
b/includes/deferred/LinksUpdate.php
index 9cd7708..d5ed250 100644
--- a/includes/deferred/LinksUpdate.php
+++ b/includes/deferred/LinksUpdate.php
@@ -218,6 +218,9 @@
$changed = $propertiesDeletes + array_diff_assoc(
$this->mProperties, $existing );
$this->invalidateProperties( $changed );
+ # Update the links table freshness for this title
+ $this->updateLinksTimestamp();
+
# Refresh links of all pages including this page
# This will be in a separate transaction
if ( $this->mRecursive ) {
@@ -855,6 +858,19 @@
return $result;
}
+
+ /**
+ * Update links table freshness
+ */
+ protected function updateLinksTimestamp() {
+ if ( $this->mId ) {
+ $this->mDb->update( 'page',
+ array( 'page_links_updated' =>
$this->mDb->timestamp() ),
+ array( 'page_id' => $this->mId ),
+ __METHOD__
+ );
+ }
+ }
}
/**
diff --git a/includes/installer/MysqlUpdater.php
b/includes/installer/MysqlUpdater.php
index cc5313a..c6762e7 100644
--- a/includes/installer/MysqlUpdater.php
+++ b/includes/installer/MysqlUpdater.php
@@ -245,6 +245,7 @@
// 1.23
array( 'addField', 'recentchanges', 'rc_source',
'patch-rc_source.sql' ),
+ array( 'addField', 'page', 'page_links_updated',
'patch-page_links_updated.sql' ),
);
}
diff --git a/includes/installer/PostgresUpdater.php
b/includes/installer/PostgresUpdater.php
index 3ecb79b..60d4eb5 100644
--- a/includes/installer/PostgresUpdater.php
+++ b/includes/installer/PostgresUpdater.php
@@ -400,6 +400,7 @@
// 1.23
array( 'addPgField', 'recentchanges', 'rc_source',
"TEXT NOT NULL DEFAULT ''" ),
+ array( 'addPgField', 'page', 'page_links_updated',
"TIMESTAMPTZ NULL" ),
);
}
diff --git a/includes/installer/SqliteUpdater.php
b/includes/installer/SqliteUpdater.php
index e0ed1ec..ba79661 100644
--- a/includes/installer/SqliteUpdater.php
+++ b/includes/installer/SqliteUpdater.php
@@ -123,6 +123,7 @@
// 1.23
array( 'addField', 'recentchanges', 'rc_source',
'patch-rc_source.sql' ),
+ array( 'addField', 'page', 'page_links_updated',
'patch-page_links_updated.sql' ),
);
}
diff --git a/includes/job/jobs/RefreshLinksJob.php
b/includes/job/jobs/RefreshLinksJob.php
index f446f64..b49977e 100644
--- a/includes/job/jobs/RefreshLinksJob.php
+++ b/includes/job/jobs/RefreshLinksJob.php
@@ -138,6 +138,10 @@
if ( isset( $this->params['rootJobTimestamp'] ) ) {
$page = WikiPage::factory( $title );
$skewedTimestamp = wfTimestamp( TS_UNIX,
$this->params['rootJobTimestamp'] ) + 5;
+ if ( $page->getLinksTimestamp() > wfTimestamp( TS_MW,
$skewedTimestamp ) ) {
+ // Something already updated the backlinks
since this job was made
+ return true;
+ }
if ( $page->getTouched() > wfTimestamp( TS_MW,
$skewedTimestamp ) ) {
$parserOptions = $page->makeParserOptions(
'canonical' );
$parserOutput =
ParserCache::singleton()->getDirty( $page, $parserOptions );
diff --git a/maintenance/archives/patch-page_links_updated.sql
b/maintenance/archives/patch-page_links_updated.sql
new file mode 100644
index 0000000..18d9e2d
--- /dev/null
+++ b/maintenance/archives/patch-page_links_updated.sql
@@ -0,0 +1,2 @@
+ALTER TABLE /*$wgDBprefix*/page
+ ADD page_links_updated varbinary(14) NULL default NULL;
diff --git a/maintenance/postgres/tables.sql b/maintenance/postgres/tables.sql
index d0d1e92..ff69241 100644
--- a/maintenance/postgres/tables.sql
+++ b/maintenance/postgres/tables.sql
@@ -80,6 +80,7 @@
page_is_new SMALLINT NOT NULL DEFAULT 0,
page_random NUMERIC(15,14) NOT NULL DEFAULT RANDOM(),
page_touched TIMESTAMPTZ,
+ page_links_updated TIMESTAMPTZ NULL,
page_latest INTEGER NOT NULL, -- FK?
page_len INTEGER NOT NULL,
page_content_model TEXT
diff --git a/maintenance/tables.sql b/maintenance/tables.sql
index 18139b2..2dde6b7 100644
--- a/maintenance/tables.sql
+++ b/maintenance/tables.sql
@@ -256,6 +256,11 @@
-- of contained templates.
page_touched binary(14) NOT NULL default '',
+ -- This timestamp is updated whenever a page is re-parsed and
+ -- it has all the link tracking tables updated for it. This is
+ -- useful for de-duplicating expensive backlink update jobs.
+ page_links_updated varbinary(14) NULL default NULL,
+
-- Handy key to revision.rev_id of the current revision.
-- This may be 0 during page creation, but that shouldn't
-- happen outside of a transaction... hopefully.
--
To view, visit https://gerrit.wikimedia.org/r/101170
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I74b6f507ef7371db92e0c3f058d38c0ca5dea9ef
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/core
Gerrit-Branch: master
Gerrit-Owner: Aaron Schulz <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
