Addshore has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/368312 )
Change subject: WB: Move injection of RC records to a separate job. ...................................................................... WB: Move injection of RC records to a separate job. Backport for 1.30.0-wmf.10 from https://gerrit.wikimedia.org/r/367328 Bug: T171370 Change-Id: I01776073e8a9fbf440e4030f3493b209beef93cb --- M composer.lock M extensions/Wikibase/client/WikibaseClient.php A extensions/Wikibase/client/includes/Changes/InjectRCRecordsJob.php M extensions/Wikibase/client/includes/Changes/WikiPageUpdater.php M extensions/Wikibase/client/includes/WikibaseClient.php A extensions/Wikibase/client/tests/phpunit/includes/Changes/InjectRCRecordsJobTest.php M extensions/Wikibase/client/tests/phpunit/includes/Changes/WikiPageUpdaterTest.php M vendor/composer/autoload_classmap.php M vendor/composer/autoload_static.php M vendor/composer/installed.json 10 files changed, 690 insertions(+), 196 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Wikidata refs/changes/12/368312/1 diff --git a/composer.lock b/composer.lock index 29b06e4..bd07d27 100644 --- a/composer.lock +++ b/composer.lock @@ -1026,7 +1026,7 @@ "support": { "issues": "https://phabricator.wikimedia.org/project/profile/1202/"; }, - "time": "2017-07-21T19:59:47+00:00" + "time": "2017-07-21T11:21:34+00:00" }, { "name": "wikibase/data-model", @@ -1484,12 +1484,12 @@ "source": { "type": "git", "url": "https://github.com/wikimedia/mediawiki-extensions-Wikibase.git";, - "reference": "48684cef3f4fb79d517a247ddea72f2e4883d07b" + "reference": "467e1cd2ddc0073c6ff91eeab42c4eae4386ecf4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/wikimedia/mediawiki-extensions-Wikibase/zipball/48684cef3f4fb79d517a247ddea72f2e4883d07b";, - "reference": "48684cef3f4fb79d517a247ddea72f2e4883d07b", + "url": "https://api.github.com/repos/wikimedia/mediawiki-extensions-Wikibase/zipball/467e1cd2ddc0073c6ff91eeab42c4eae4386ecf4";, + "reference": "467e1cd2ddc0073c6ff91eeab42c4eae4386ecf4", "shasum": "" }, "require": { @@ -1565,7 +1565,7 @@ "wikibaserepo", "wikidata" ], - "time": "2017-07-24T00:47:16+00:00" + "time": "2017-07-27T17:56:33+00:00" }, { "name": "wikibase/wikimedia-badges", diff --git a/extensions/Wikibase/client/WikibaseClient.php b/extensions/Wikibase/client/WikibaseClient.php index fd0fe88..012905d 100644 --- a/extensions/Wikibase/client/WikibaseClient.php +++ b/extensions/Wikibase/client/WikibaseClient.php @@ -182,6 +182,24 @@ // job classes $wgJobClasses['wikibase-addUsagesForPage'] = Wikibase\Client\Store\AddUsagesForPageJob::class; $wgJobClasses['ChangeNotification'] = Wikibase\Client\ChangeNotificationJob::class; + $wgJobClasses['wikibase-InjectRCRecords'] = function ( Title $unused, array $params ) { + $mwServices = MediaWiki\MediaWikiServices::getInstance(); + $wbServices = Wikibase\Client\WikibaseClient::getDefaultInstance(); + + $job = new Wikibase\Client\Changes\InjectRCRecordsJob( + $mwServices->getDBLoadBalancerFactory(), + $wbServices->getStore()->getEntityChangeLookup(), + $wbServices->getRecentChangeFactory(), + $params + ); + + $job->setRecentChangesDuplicateDetector( $wbServices->getStore()->getRecentChangesDuplicateDetector() ); + + $job->setLogger( MediaWiki\Logger\LoggerFactory::getInstance( 'wikibase.client.pageupdates' ) ); + $job->setStats( $mwServices->getStatsdDataFactory() ); + + return $job; + }; // api modules $wgAPIMetaModules['wikibase'] = array( @@ -226,8 +244,8 @@ $wgAPIPropModules['wbentityusage'] = [ 'class' => Wikibase\Client\Api\ApiPropsEntityUsage::class, 'factory' => function ( ApiQuery $query, $moduleName ) { - $repoLinker = \Wikibase\Client\WikibaseClient::getDefaultInstance()->newRepoLinker(); - return new \Wikibase\Client\Api\ApiPropsEntityUsage( + $repoLinker = Wikibase\Client\WikibaseClient::getDefaultInstance()->newRepoLinker(); + return new Wikibase\Client\Api\ApiPropsEntityUsage( $query, $moduleName, $repoLinker diff --git a/extensions/Wikibase/client/includes/Changes/InjectRCRecordsJob.php b/extensions/Wikibase/client/includes/Changes/InjectRCRecordsJob.php new file mode 100644 index 0000000..54c4781 --- /dev/null +++ b/extensions/Wikibase/client/includes/Changes/InjectRCRecordsJob.php @@ -0,0 +1,272 @@ +<?php + +namespace Wikibase\Client\Changes; + +use Job; +use JobSpecification; +use Liuggio\StatsdClient\Factory\StatsdDataFactoryInterface; +use Psr\Log\LoggerInterface; +use Psr\Log\NullLogger; +use Title; +use Wikibase\Client\Changes\ChangeHandler; +use Wikibase\Client\RecentChanges\RecentChangeFactory; +use Wikibase\Client\RecentChanges\RecentChangesDuplicateDetector; +use Wikibase\Client\Store\TitleFactory; +use Wikibase\Client\WikibaseClient; +use Wikibase\EntityChange; +use Wikibase\Lib\Store\Sql\EntityChangeLookup; +use Wikimedia\Assert\Assert; +use Wikimedia\Rdbms\LBFactory; + +/** + * Job for injecting RecentChange records representing changes on the Wikibase repository. + * + * @see docs/change-propagation.wiki for an overview of the change propagation mechanism. + * + * @license GPL-2.0+ + * @author Daniel Kinzler + */ +class InjectRCRecordsJob extends Job { + + /** + * @var LBFactory + */ + private $lbFactory; + + /** + * @var EntityChangeLookup + */ + private $changeLookup; + + /** + * @var RecentChangeFactory + */ + private $rcFactory; + + /** + * @var RecentChangesDuplicateDetector|null + */ + private $rcDuplicateDetector = null; + + /** + * @var TitleFactory + */ + private $titleFactory; + + /** + * @var LoggerInterface + */ + private $logger; + + /** + * @var StatsdDataFactoryInterface|null + */ + private $stats = null; + + /** + * @var int Batch size for database operations + */ + private $dbBatchSize = 100; + + /** + * @param Title[] $titles + * @param EntityChange $change + * + * @return JobSpecification + */ + public static function makeJobSpecification( array $titles, EntityChange $change ) { + $pages = []; + + foreach ( $titles as $t ) { + $id = $t->getArticleId(); + $pages[$id] = [ $t->getNamespace(), $t->getDBkey() ]; + } + + $params = [ + 'change' => $change->getId(), + 'pages' => $pages + ]; + + return new JobSpecification( + 'wikibase-InjectRCRecords', + $params + ); + + } + + /** + * Constructs an InjectRCRecordsJob for injecting a change into the recentchanges feed + * for the given pages. + * + * @param LBFactory $lbFactory + * @param EntityChangeLookup $changeLookup + * @param RecentChangeFactory $rcFactory + * @param array $params Needs to have two keys: "change": the id of the change, + * "pages": array of pages, represented as $pageId => [ $namespace, $dbKey ]. + */ + public function __construct( + LBFactory $lbFactory, + EntityChangeLookup $changeLookup, + RecentChangeFactory $rcFactory, + array $params + ) { + $title = Title::makeTitle( NS_SPECIAL, 'Badtitle/' . __CLASS__ ); + parent::__construct( 'wikibase-InjectRCRecords', $title, $params ); + + Assert::parameter( + isset( $params['change'] ), + '$params', + '$params[\'change\'] not set.' + ); + Assert::parameter( + isset( $params['pages'] ), + '$params', + '$params[\'pages\'] not set.' + ); + Assert::parameterElementType( + 'array', + $params['pages'], + '$params[\'pages\']' + ); + + $this->lbFactory = $lbFactory; + $this->changeLookup = $changeLookup; + $this->rcFactory = $rcFactory; + + $this->titleFactory = new TitleFactory(); + $this->logger = new NullLogger(); + } + + /** + * @param RecentChangesDuplicateDetector $rcDuplicateDetector + */ + public function setRecentChangesDuplicateDetector( RecentChangesDuplicateDetector $rcDuplicateDetector ) { + $this->rcDuplicateDetector = $rcDuplicateDetector; + } + + /** + * @param TitleFactory $titleFactory + */ + public function setTitleFactory( TitleFactory $titleFactory ) { + $this->titleFactory = $titleFactory; + } + + /** + * @param LoggerInterface $logger + */ + public function setLogger( LoggerInterface $logger ) { + $this->logger = $logger; + } + + /** + * @param StatsdDataFactoryInterface $stats + */ + public function setStats( StatsdDataFactoryInterface $stats ) { + $this->stats = $stats; + } + + /** + * @param int $dbBatchSize + */ + public function setDbBatchSize( $dbBatchSize ) { + Assert::parameterType( 'integer', $dbBatchSize, '$dbBatchSize' ); + $this->dbBatchSize = $dbBatchSize; + } + + /** + * Returns the change that should be processed. + * + * EntityChange objects are loaded using a EntityChangeLookup. + * + * @return EntityChange|null the change to process (or none). + */ + private function getChange() { + $params = $this->getParams(); + $changeId = $params['change']; + + $this->logger->debug( __FUNCTION__ . ": loading change $changeId." ); + + $changes = $this->changeLookup->loadByChangeIds( [ $changeId ] ); + + $change = reset( $changes ); + + if ( !$change ) { + $this->logger->error( __FUNCTION__ . ": failed to load change $changeId." ); + } + + return $change; + } + + /** + * Returns the list of Titles to inject RC entries for. + * + * @return Title[] + */ + private function getTitles() { + $params = $this->getParams(); + $pages = $params['pages']; + + $titles = []; + + foreach ( $pages as $pageId => list( $namespace, $dbKey ) ) { + $titles[] = $this->titleFactory->makeTitle( $namespace, $dbKey ); + } + + return $titles; + } + + /** + * @return bool success + */ + public function run() { + $change = $this->getChange(); + $titles = $this->getTitles(); + + if ( !$change || $titles === [] ) { + return false; + } + + $rcAttribs = $this->rcFactory->prepareChangeAttributes( $change ); + + $c = 0; + $trxToken = $this->lbFactory->getEmptyTransactionTicket( __METHOD__ ); + + foreach ( $titles as $title ) { + if ( !$title->exists() ) { + continue; + } + + $rc = $this->rcFactory->newRecentChange( $change, $title, $rcAttribs ); + + if ( $this->rcDuplicateDetector + && $this->rcDuplicateDetector->changeExists( $rc ) + ) { + $this->logger->debug( __FUNCTION__ . ": skipping duplicate RC entry for " . $title->getFullText() ); + } else { + $this->logger->debug( __FUNCTION__ . ": saving RC entry for " . $title->getFullText() ); + $rc->save(); + } + + if ( ++$c >= $this->dbBatchSize ) { + $this->lbFactory->commitAndWaitForReplication( __METHOD__, $trxToken ); + $trxToken = $this->lbFactory->getEmptyTransactionTicket( __METHOD__ ); + $c = 0; + } + } + + if ( $c > 0 ) { + $this->lbFactory->commitAndWaitForReplication( __METHOD__, $trxToken ); + } + + $this->incrementStats( 'InjectRCRecords.run.titles', count( $titles ) ); + + return true; + } + + private function incrementStats( $updateType, $delta ) { + if ( $this->stats ) { + $this->stats->updateCount( 'wikibase.client.pageupdates.' . $updateType, $delta ); + } + } + +} diff --git a/extensions/Wikibase/client/includes/Changes/WikiPageUpdater.php b/extensions/Wikibase/client/includes/Changes/WikiPageUpdater.php index 6fd4f03..3ab41b1 100644 --- a/extensions/Wikibase/client/includes/Changes/WikiPageUpdater.php +++ b/extensions/Wikibase/client/includes/Changes/WikiPageUpdater.php @@ -155,40 +155,16 @@ * @param EntityChange $change */ public function injectRCRecords( array $titles, EntityChange $change ) { - $rcAttribs = $this->recentChangeFactory->prepareChangeAttributes( $change ); - - $c = 0; - $trxToken = $this->LBFactory->getEmptyTransactionTicket( __METHOD__ ); - - // TODO: do this via the job queue, in batches, see T107722 - foreach ( $titles as $title ) { - if ( !$title->exists() ) { - continue; - } - - $rc = $this->recentChangeFactory->newRecentChange( $change, $title, $rcAttribs ); - - if ( $this->recentChangesDuplicateDetector - && $this->recentChangesDuplicateDetector->changeExists( $rc ) - ) { - wfDebugLog( __CLASS__, __FUNCTION__ . ": skipping duplicate RC entry for " . $title->getFullText() ); - } else { - wfDebugLog( __CLASS__, __FUNCTION__ . ": saving RC entry for " . $title->getFullText() ); - $rc->save(); - } - - if ( ++$c >= $this->dbBatchSize ) { - $this->LBFactory->commitAndWaitForReplication( __METHOD__, $trxToken ); - $trxToken = $this->LBFactory->getEmptyTransactionTicket( __METHOD__ ); - $c = 0; - } + if ( $titles === [] ) { + return; } - if ( $c > 0 ) { - $this->LBFactory->commitAndWaitForReplication( __METHOD__, $trxToken ); - } + $jobSpec = InjectRCRecordsJob::makeJobSpecification( $titles, $change ); - $this->incrementStats( 'InjectRCRecords', count( $titles ) ); + $this->jobQueueGroup->lazyPush( $jobSpec ); + + $this->incrementStats( 'InjectRCRecords.jobs', 1 ); + $this->incrementStats( 'InjectRCRecords.titles', count( $titles ) ); } } diff --git a/extensions/Wikibase/client/includes/WikibaseClient.php b/extensions/Wikibase/client/includes/WikibaseClient.php index ddc6dad..6b2d353 100644 --- a/extensions/Wikibase/client/includes/WikibaseClient.php +++ b/extensions/Wikibase/client/includes/WikibaseClient.php @@ -1186,7 +1186,7 @@ /** * @return RecentChangeFactory */ - private function getRecentChangeFactory() { + public function getRecentChangeFactory() { return new RecentChangeFactory( $this->getContentLanguage(), new SiteLinkCommentCreator( diff --git a/extensions/Wikibase/client/tests/phpunit/includes/Changes/InjectRCRecordsJobTest.php b/extensions/Wikibase/client/tests/phpunit/includes/Changes/InjectRCRecordsJobTest.php new file mode 100644 index 0000000..e182b76 --- /dev/null +++ b/extensions/Wikibase/client/tests/phpunit/includes/Changes/InjectRCRecordsJobTest.php @@ -0,0 +1,236 @@ +<?php + +namespace Wikibase\Client\Tests\Changes; + +use PHPUnit_Framework_MockObject_MockObject; +use RecentChange; +use Title; +use Wikibase\Client\Changes\InjectRCRecordsJob; +use Wikibase\Client\RecentChanges\RecentChangeFactory; +use Wikibase\Client\RecentChanges\RecentChangesDuplicateDetector; +use Wikibase\Client\Store\TitleFactory; +use Wikibase\EntityChange; +use Wikibase\Lib\Store\Sql\EntityChangeLookup; +use Wikimedia\Rdbms\LBFactory; + +/** + * @covers Wikibase\Client\Changes\InjectRCRecordsJob + * + * @group Wikibase + * @group WikibaseClient + * @group WikibaseChange + * + * @group Database + * + * @license GPL-2.0+ + * @author Daniel Kinzler + */ +class InjectRCRecordsJobTest extends \MediaWikiTestCase { + + /** + * @return RecentChangeFactory|PHPUnit_Framework_MockObject_MockObject + */ + private function getRCFactoryMock() { + $rcFactory = $this->getMockBuilder( RecentChangeFactory::class ) + ->disableOriginalConstructor() + ->getMock(); + + $rcFactory->expects( $this->any() ) + ->method( 'prepareChangeAttributes' ) + ->will( $this->returnValue( [] ) ); + + return $rcFactory; + } + + /** + * @param EntityChange $change + * + * @return PHPUnit_Framework_MockObject_MockObject|EntityChangeLookup + */ + private function getEntityChangeLookupMock( EntityChange $change ) { + $changeLookup = $this->getMockBuilder( EntityChangeLookup::class ) + ->disableOriginalConstructor() + ->getMock(); + + $changeLookup->expects( $this->any() ) + ->method( 'loadByChangeIds' ) + ->with( [ $change->getId() ] ) + ->will( $this->returnValue( [ $change ] ) ); + + return $changeLookup; + } + + /** + * @return RecentChangesDuplicateDetector|PHPUnit_Framework_MockObject_MockObject + */ + private function getRCDupeDetectorMock() { + $rcDupeDetector = $this->getMockBuilder( RecentChangesDuplicateDetector::class ) + ->disableOriginalConstructor() + ->getMock(); + + return $rcDupeDetector; + } + + /** + * @return TitleFactory|PHPUnit_Framework_MockObject_MockObject + */ + private function getTitleFactoryMock() { + $titleFactory = $this->getMock( TitleFactory::class ); + + $id = 200; + $titleFactory->expects( $this->any() ) + ->method( 'makeTitle' ) + ->will( $this->returnCallback( function( $ns, $text ) use ( &$id ) { + return $this->getTitleMock( $text, $id++ ); + } ) ); + + return $titleFactory; + } + + /** + * @param string $text + * @param int $id + * + * @return Title|PHPUnit_Framework_MockObject_MockObject + */ + private function getTitleMock( $text, $id = 23 ) { + $title = $this->getMockBuilder( Title::class ) + ->disableOriginalConstructor() + ->getMock(); + + $title->expects( $this->any() ) + ->method( 'getArticleID' ) + ->will( $this->returnValue( $id ) ); + + $title->expects( $this->any() ) + ->method( 'exists' ) + ->will( $this->returnValue( true ) ); + + $title->expects( $this->any() ) + ->method( 'getDBkey' ) + ->will( $this->returnValue( $text ) ); + + $title->expects( $this->any() ) + ->method( 'getNamespace' ) + ->will( $this->returnValue( 0 ) ); + + return $title; + } + + /** + * @param int $id + * + * @return PHPUnit_Framework_MockObject_MockObject|EntityChange + */ + private function getEntityChangeMock( $id = 77 ) { + $change = $this->getMockBuilder( EntityChange::class ) + ->disableOriginalConstructor() + ->getMock(); + + $change->expects( $this->any() ) + ->method( 'getId' ) + ->will( $this->returnValue( $id ) ); + + return $change; + } + + /** + * @return RecentChange|PHPUnit_Framework_MockObject_MockObject + */ + private function getRecentChangeMock() { + $change = $this->getMockBuilder( RecentChange::class ) + ->disableOriginalConstructor() + ->getMock(); + + return $change; + } + + /** + * @return LBFactory|PHPUnit_Framework_MockObject_MockObject + */ + private function getLBFactoryMock() { + $LBFactory = $this->getMockBuilder( LBFactory::class ) + ->disableOriginalConstructor() + ->getMock(); + + return $LBFactory; + } + + public function testRun() { + $title = $this->getTitleMock( 'Foo', 21 ); + $change = $this->getEntityChangeMock( 17 ); + $rc = $this->getRecentChangeMock(); + + $changeLookup = $this->getEntityChangeLookupMock( $change ); + + $rcFactory = $this->getRCFactoryMock(); + + $rcFactory->expects( $this->once() ) + ->method( 'newRecentChange' ) + ->with( $change, $title, [] ) + ->will( $this->returnValue( $rc ) ); + + $rcDupeDetector = $this->getRCDupeDetectorMock(); + + $rcDupeDetector->expects( $this->once() ) + ->method( 'changeExists' ) + ->with( $rc ); + + $params = [ + 'change' => $change->getId(), + 'pages' => [ + 21 => [ 0, 'Foo' ] + ] + ]; + + $job = new InjectRCRecordsJob( + $this->getLBFactoryMock(), + $changeLookup, + $rcFactory, + $params + ); + + $job->setTitleFactory( $this->getTitleFactoryMock() ); + $job->setRecentChangesDuplicateDetector( $rcDupeDetector ); + + $job->run(); + } + + public function testRun_batch() { + $change = $this->getEntityChangeMock(); + $rc = $this->getRecentChangeMock(); + $changeLookup = $this->getEntityChangeLookupMock( $change ); + + $rcFactory = $this->getRCFactoryMock(); + + $rcFactory->expects( $this->any() ) + ->method( 'newRecentChange' ) + ->will( $this->returnValue( $rc ) ); + + $lbFactory = $this->getLBFactoryMock(); + $lbFactory->expects( $this->exactly( 2 ) ) + ->method( 'commitAndWaitForReplication' ); + + $params = [ + 'change' => $change->getId(), + 'pages' => [ + 21 => [ 0, 'Foo' ], + 22 => [ 0, 'Bar' ], + 23 => [ 0, 'Cuzz' ], + ] + ]; + + $job = new InjectRCRecordsJob( + $lbFactory, + $changeLookup, + $rcFactory, + $params + ); + + $job->setTitleFactory( $this->getTitleFactoryMock() ); + $job->setDbBatchSize( 2 ); + + $job->run(); + } + +} diff --git a/extensions/Wikibase/client/tests/phpunit/includes/Changes/WikiPageUpdaterTest.php b/extensions/Wikibase/client/tests/phpunit/includes/Changes/WikiPageUpdaterTest.php index f73f61f..2be8d28 100644 --- a/extensions/Wikibase/client/tests/phpunit/includes/Changes/WikiPageUpdaterTest.php +++ b/extensions/Wikibase/client/tests/phpunit/includes/Changes/WikiPageUpdaterTest.php @@ -2,9 +2,10 @@ namespace Wikibase\Client\Tests\Changes; +use IJobSpecification; use Job; use JobQueueGroup; -use RecentChange; +use PHPUnit_Framework_MockObject_MockObject; use RefreshLinksJob; use Title; use Wikibase\Client\Changes\WikiPageUpdater; @@ -28,7 +29,7 @@ class WikiPageUpdaterTest extends \MediaWikiTestCase { /** - * @return JobQueueGroup + * @return JobQueueGroup|PHPUnit_Framework_MockObject_MockObject */ private function getJobQueueGroupMock() { $jobQueueGroup = $this->getMockBuilder( JobQueueGroup::class ) @@ -66,17 +67,18 @@ /** * @param string $text + * @param int $id * - * @return Title + * @return Title|PHPUnit_Framework_MockObject_MockObject */ - private function getTitleMock( $text ) { + private function getTitleMock( $text, $id = 23 ) { $title = $this->getMockBuilder( Title::class ) ->disableOriginalConstructor() ->getMock(); $title->expects( $this->any() ) ->method( 'getArticleID' ) - ->will( $this->returnValue( 23 ) ); + ->will( $this->returnValue( $id ) ); $title->expects( $this->any() ) ->method( 'exists' ) @@ -86,27 +88,38 @@ ->method( 'getPrefixedDBkey' ) ->will( $this->returnValue( $text ) ); + $title->expects( $this->any() ) + ->method( 'getDBkey' ) + ->will( $this->returnValue( $text ) ); + + $title->expects( $this->any() ) + ->method( 'getText' ) + ->will( $this->returnValue( $text ) ); + + $title->expects( $this->any() ) + ->method( 'getNamespace' ) + ->will( $this->returnValue( 0 ) ); + + $title->expects( $this->any() ) + ->method( 'getNsText' ) + ->will( $this->returnValue( '' ) ); + return $title; } /** - * @return EntityChange + * @param int $id + * + * @return PHPUnit_Framework_MockObject_MockObject|EntityChange */ - private function getEntityChangeMock() { + private function getEntityChangeMock( $id = 77 ) { $change = $this->getMockBuilder( EntityChange::class ) ->disableOriginalConstructor() ->getMock(); - return $change; - } - - /** - * @return RecentChange - */ - private function getRecentChangeMock() { - $change = $this->getMockBuilder( RecentChange::class ) - ->disableOriginalConstructor() - ->getMock(); + $change->expects( $this->any() ) + ->method( 'getId' ) + ->will( $this->returnValue( $id ) ); return $change; } @@ -196,68 +209,43 @@ } public function testInjectRCRecords() { - $title = $this->getTitleMock( 'Foo' ); + $titleFoo = $this->getTitleMock( 'Foo', 21 ); + $titleBar = $this->getTitleMock( 'Bar', 22 ); + $titleCuzz = $this->getTitleMock( 'Cuzz', 23 ); + $change = $this->getEntityChangeMock(); - $rc = $this->getRecentChangeMock(); - $rcFactory = $this->getRCFactoryMock(); + $jobQueueGroup = $this->getJobQueueGroupMock(); - $rcFactory->expects( $this->once() ) - ->method( 'newRecentChange' ) - ->with( $change, $title, [] ) - ->will( $this->returnValue( $rc ) ); + $pages = []; + $jobQueueGroup->expects( $this->atLeastOnce() ) + ->method( 'lazyPush' ) + ->will( $this->returnCallback( function( IJobSpecification $job ) use ( &$pages, $change ) { + $params = $job->getParams(); - $rcDupeDetector = $this->getRCDupeDetectorMock(); + $this->assertArrayHasKey( 'change', $params, '$params["change"]' ); + $this->assertArrayHasKey( 'pages', $params, '$params["pages"]' ); - $rcDupeDetector->expects( $this->once() ) - ->method( 'changeExists' ) - ->with( $rc ); + $this->assertSame( $change->getId(), $params["change"] ); + + $pages += $params['pages']; // addition uses keys, array_merge does not + } ) ); $updater = new WikiPageUpdater( - $this->getJobQueueGroupMock(), - $rcFactory, + $jobQueueGroup, + $this->getRCFactoryMock(), $this->getLBFactoryMock(), - $rcDupeDetector + $this->getRCDupeDetectorMock() ); $updater->injectRCRecords( [ - $title + $titleFoo, $titleBar, $titleCuzz, ], $change ); - } - public function testInjectRCRecords_batch() { - $titleFoo = $this->getTitleMock( 'Foo' ); - $titleBar = $this->getTitleMock( 'Bar' ); - $titleCuzz = $this->getTitleMock( 'Cuzz' ); - - $change = $this->getEntityChangeMock(); - $rc = $this->getRecentChangeMock(); - - $rcFactory = $this->getRCFactoryMock(); - - $rcFactory->expects( $this->any() ) - ->method( 'newRecentChange' ) - ->will( $this->returnValue( $rc ) ); - - $rcDupeDetector = $this->getRCDupeDetectorMock(); - - $lbFactory = $this->getLBFactoryMock(); - $lbFactory->expects( $this->exactly( 2 ) ) - ->method( 'commitAndWaitForReplication' ); - - $updater = new WikiPageUpdater( - $this->getJobQueueGroupMock(), - $rcFactory, - $lbFactory, - $rcDupeDetector - ); - - $updater->setDbBatchSize( 2 ); - - $updater->injectRCRecords( - [ $titleFoo, $titleBar, $titleCuzz ], - $change - ); + $this->assertEquals( [ 21, 22, 23 ], array_keys( $pages ) ); + $this->assertEquals( [ 0, 'Foo' ], $pages[21], '$pages[21]' ); + $this->assertEquals( [ 0, 'Bar' ], $pages[22], '$pages[22]' ); + $this->assertEquals( [ 0, 'Cuzz' ], $pages[23], '$pages[23]' ); } } diff --git a/vendor/composer/autoload_classmap.php b/vendor/composer/autoload_classmap.php index e55d96f..37540bf 100644 --- a/vendor/composer/autoload_classmap.php +++ b/vendor/composer/autoload_classmap.php @@ -326,6 +326,7 @@ 'Wikibase\\Client\\Changes\\AffectedPagesFinder' => $baseDir . '/extensions/Wikibase/client/includes/Changes/AffectedPagesFinder.php', 'Wikibase\\Client\\Changes\\ChangeHandler' => $baseDir . '/extensions/Wikibase/client/includes/Changes/ChangeHandler.php', 'Wikibase\\Client\\Changes\\ChangeRunCoalescer' => $baseDir . '/extensions/Wikibase/client/includes/Changes/ChangeRunCoalescer.php', + 'Wikibase\\Client\\Changes\\InjectRCRecordsJob' => $baseDir . '/extensions/Wikibase/client/includes/Changes/InjectRCRecordsJob.php', 'Wikibase\\Client\\Changes\\PageUpdater' => $baseDir . '/extensions/Wikibase/client/includes/Changes/PageUpdater.php', 'Wikibase\\Client\\Changes\\WikiPageUpdater' => $baseDir . '/extensions/Wikibase/client/includes/Changes/WikiPageUpdater.php', 'Wikibase\\Client\\DataAccess\\ClientSiteLinkTitleLookup' => $baseDir . '/extensions/Wikibase/client/includes/DataAccess/ClientSiteLinkTitleLookup.php', @@ -406,6 +407,7 @@ 'Wikibase\\Client\\Tests\\Changes\\AffectedPagesFinderTest' => $baseDir . '/extensions/Wikibase/client/tests/phpunit/includes/Changes/AffectedPagesFinderTest.php', 'Wikibase\\Client\\Tests\\Changes\\ChangeHandlerTest' => $baseDir . '/extensions/Wikibase/client/tests/phpunit/includes/Changes/ChangeHandlerTest.php', 'Wikibase\\Client\\Tests\\Changes\\ChangeRunCoalescerTest' => $baseDir . '/extensions/Wikibase/client/tests/phpunit/includes/Changes/ChangeRunCoalescerTest.php', + 'Wikibase\\Client\\Tests\\Changes\\InjectRCRecordsJobTest' => $baseDir . '/extensions/Wikibase/client/tests/phpunit/includes/Changes/InjectRCRecordsJobTest.php', 'Wikibase\\Client\\Tests\\Changes\\MockPageUpdater' => $baseDir . '/extensions/Wikibase/client/tests/phpunit/includes/Changes/MockPageUpdater.php', 'Wikibase\\Client\\Tests\\Changes\\WikiPageUpdaterTest' => $baseDir . '/extensions/Wikibase/client/tests/phpunit/includes/Changes/WikiPageUpdaterTest.php', 'Wikibase\\Client\\Tests\\ClientParserOutputDataUpdaterTest' => $baseDir . '/extensions/Wikibase/client/tests/phpunit/includes/ClientParserOutputDataUpdaterTest.php', diff --git a/vendor/composer/autoload_static.php b/vendor/composer/autoload_static.php index e85093b..c5c6ccf 100644 --- a/vendor/composer/autoload_static.php +++ b/vendor/composer/autoload_static.php @@ -618,6 +618,7 @@ 'Wikibase\\Client\\Changes\\AffectedPagesFinder' => __DIR__ . '/../..' . '/extensions/Wikibase/client/includes/Changes/AffectedPagesFinder.php', 'Wikibase\\Client\\Changes\\ChangeHandler' => __DIR__ . '/../..' . '/extensions/Wikibase/client/includes/Changes/ChangeHandler.php', 'Wikibase\\Client\\Changes\\ChangeRunCoalescer' => __DIR__ . '/../..' . '/extensions/Wikibase/client/includes/Changes/ChangeRunCoalescer.php', + 'Wikibase\\Client\\Changes\\InjectRCRecordsJob' => __DIR__ . '/../..' . '/extensions/Wikibase/client/includes/Changes/InjectRCRecordsJob.php', 'Wikibase\\Client\\Changes\\PageUpdater' => __DIR__ . '/../..' . '/extensions/Wikibase/client/includes/Changes/PageUpdater.php', 'Wikibase\\Client\\Changes\\WikiPageUpdater' => __DIR__ . '/../..' . '/extensions/Wikibase/client/includes/Changes/WikiPageUpdater.php', 'Wikibase\\Client\\DataAccess\\ClientSiteLinkTitleLookup' => __DIR__ . '/../..' . '/extensions/Wikibase/client/includes/DataAccess/ClientSiteLinkTitleLookup.php', @@ -698,6 +699,7 @@ 'Wikibase\\Client\\Tests\\Changes\\AffectedPagesFinderTest' => __DIR__ . '/../..' . '/extensions/Wikibase/client/tests/phpunit/includes/Changes/AffectedPagesFinderTest.php', 'Wikibase\\Client\\Tests\\Changes\\ChangeHandlerTest' => __DIR__ . '/../..' . '/extensions/Wikibase/client/tests/phpunit/includes/Changes/ChangeHandlerTest.php', 'Wikibase\\Client\\Tests\\Changes\\ChangeRunCoalescerTest' => __DIR__ . '/../..' . '/extensions/Wikibase/client/tests/phpunit/includes/Changes/ChangeRunCoalescerTest.php', + 'Wikibase\\Client\\Tests\\Changes\\InjectRCRecordsJobTest' => __DIR__ . '/../..' . '/extensions/Wikibase/client/tests/phpunit/includes/Changes/InjectRCRecordsJobTest.php', 'Wikibase\\Client\\Tests\\Changes\\MockPageUpdater' => __DIR__ . '/../..' . '/extensions/Wikibase/client/tests/phpunit/includes/Changes/MockPageUpdater.php', 'Wikibase\\Client\\Tests\\Changes\\WikiPageUpdaterTest' => __DIR__ . '/../..' . '/extensions/Wikibase/client/tests/phpunit/includes/Changes/WikiPageUpdaterTest.php', 'Wikibase\\Client\\Tests\\ClientParserOutputDataUpdaterTest' => __DIR__ . '/../..' . '/extensions/Wikibase/client/tests/phpunit/includes/ClientParserOutputDataUpdaterTest.php', diff --git a/vendor/composer/installed.json b/vendor/composer/installed.json index b03f473..3d8229d 100644 --- a/vendor/composer/installed.json +++ b/vendor/composer/installed.json @@ -1617,97 +1617,6 @@ } }, { - "name": "wikibase/wikibase", - "version": "dev-wmf/1.30.0-wmf.10", - "version_normalized": "dev-wmf/1.30.0-wmf.10", - "source": { - "type": "git", - "url": "https://github.com/wikimedia/mediawiki-extensions-Wikibase.git";, - "reference": "48684cef3f4fb79d517a247ddea72f2e4883d07b" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/wikimedia/mediawiki-extensions-Wikibase/zipball/48684cef3f4fb79d517a247ddea72f2e4883d07b";, - "reference": "48684cef3f4fb79d517a247ddea72f2e4883d07b", - "shasum": "" - }, - "require": { - "data-values/common": "^0.3.0", - "data-values/data-types": "^1.0.0", - "data-values/data-values": "^1.0.0", - "data-values/geo": "^2.0.1", - "data-values/interfaces": "^0.2.0|^0.1.5", - "data-values/javascript": "^0.8.3", - "data-values/number": "^0.8.3", - "data-values/serialization": "^1.2.1", - "data-values/time": "^0.8.4", - "data-values/value-view": "^0.19.2", - "diff/diff": "^2.0.0", - "php": ">=5.5.9", - "wikibase/data-model": "^7.0.0", - "wikibase/data-model-javascript": "^3.0.1", - "wikibase/data-model-serialization": "^2.1.0", - "wikibase/data-model-services": "^3.7.0", - "wikibase/internal-serialization": "^2.2.0", - "wikibase/javascript-api": "^2.2.2", - "wikibase/serialization-javascript": "^2.0.8", - "wikimedia/purtle": "^1.0.3" - }, - "conflict": { - "mediawiki/mediawiki": "<1.25" - }, - "require-dev": { - "jakub-onderka/php-console-highlighter": "0.3.2", - "jakub-onderka/php-parallel-lint": ">=0.3 <0.10", - "wikibase/wikibase-codesniffer": "^0.1.0" - }, - "time": "2017-07-24T00:47:16+00:00", - "type": "mediawiki-extension", - "installation-source": "dist", - "autoload": { - "classmap": [ - "client/WikibaseClient.hooks.php", - "client/tests/phpunit/MockClientStore.php", - "lib/includes/", - "lib/WikibaseLib.hooks.php", - "lib/maintenance/", - "repo/includes/", - "repo/maintenance/", - "repo/Wikibase.hooks.php" - ], - "psr-4": { - "Wikibase\\Client\\": "client/includes", - "Wikibase\\Client\\Tests\\": "client/tests/phpunit/includes", - "Wikibase\\DataAccess\\": "data-access/src", - "Wikibase\\DataAccess\\Tests\\": "data-access/tests/phpunit", - "Wikibase\\Lib\\": "lib/includes", - "Wikibase\\Lib\\Tests\\": "lib/tests/phpunit", - "Wikibase\\Repo\\": "repo/includes", - "Wikibase\\Repo\\Tests\\": "repo/tests/phpunit/includes", - "Wikibase\\View\\": "view/src", - "Wikibase\\View\\Tests\\": "view/tests/phpunit" - } - }, - "notification-url": "https://packagist.org/downloads/";, - "license": [ - "GPL-2.0+" - ], - "authors": [ - { - "name": "The Wikidata team" - } - ], - "description": "Structured data repository for MediaWiki", - "homepage": "http://wikiba.se";, - "keywords": [ - "wikibase", - "wikibaseclient", - "wikibaselib", - "wikibaserepo", - "wikidata" - ] - }, - { "name": "wikibase/constraints", "version": "dev-wmf/1.30.0-wmf.10", "version_normalized": "dev-wmf/1.30.0-wmf.10", @@ -1777,5 +1686,96 @@ "support": { "issues": "https://phabricator.wikimedia.org/project/profile/1202/"; } + }, + { + "name": "wikibase/wikibase", + "version": "dev-wmf/1.30.0-wmf.10", + "version_normalized": "dev-wmf/1.30.0-wmf.10", + "source": { + "type": "git", + "url": "https://github.com/wikimedia/mediawiki-extensions-Wikibase.git";, + "reference": "467e1cd2ddc0073c6ff91eeab42c4eae4386ecf4" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/wikimedia/mediawiki-extensions-Wikibase/zipball/467e1cd2ddc0073c6ff91eeab42c4eae4386ecf4";, + "reference": "467e1cd2ddc0073c6ff91eeab42c4eae4386ecf4", + "shasum": "" + }, + "require": { + "data-values/common": "^0.3.0", + "data-values/data-types": "^1.0.0", + "data-values/data-values": "^1.0.0", + "data-values/geo": "^2.0.1", + "data-values/interfaces": "^0.2.0|^0.1.5", + "data-values/javascript": "^0.8.3", + "data-values/number": "^0.8.3", + "data-values/serialization": "^1.2.1", + "data-values/time": "^0.8.4", + "data-values/value-view": "^0.19.2", + "diff/diff": "^2.0.0", + "php": ">=5.5.9", + "wikibase/data-model": "^7.0.0", + "wikibase/data-model-javascript": "^3.0.1", + "wikibase/data-model-serialization": "^2.1.0", + "wikibase/data-model-services": "^3.7.0", + "wikibase/internal-serialization": "^2.2.0", + "wikibase/javascript-api": "^2.2.2", + "wikibase/serialization-javascript": "^2.0.8", + "wikimedia/purtle": "^1.0.3" + }, + "conflict": { + "mediawiki/mediawiki": "<1.25" + }, + "require-dev": { + "jakub-onderka/php-console-highlighter": "0.3.2", + "jakub-onderka/php-parallel-lint": ">=0.3 <0.10", + "wikibase/wikibase-codesniffer": "^0.1.0" + }, + "time": "2017-07-27T17:56:33+00:00", + "type": "mediawiki-extension", + "installation-source": "dist", + "autoload": { + "classmap": [ + "client/WikibaseClient.hooks.php", + "client/tests/phpunit/MockClientStore.php", + "lib/includes/", + "lib/WikibaseLib.hooks.php", + "lib/maintenance/", + "repo/includes/", + "repo/maintenance/", + "repo/Wikibase.hooks.php" + ], + "psr-4": { + "Wikibase\\Client\\": "client/includes", + "Wikibase\\Client\\Tests\\": "client/tests/phpunit/includes", + "Wikibase\\DataAccess\\": "data-access/src", + "Wikibase\\DataAccess\\Tests\\": "data-access/tests/phpunit", + "Wikibase\\Lib\\": "lib/includes", + "Wikibase\\Lib\\Tests\\": "lib/tests/phpunit", + "Wikibase\\Repo\\": "repo/includes", + "Wikibase\\Repo\\Tests\\": "repo/tests/phpunit/includes", + "Wikibase\\View\\": "view/src", + "Wikibase\\View\\Tests\\": "view/tests/phpunit" + } + }, + "notification-url": "https://packagist.org/downloads/";, + "license": [ + "GPL-2.0+" + ], + "authors": [ + { + "name": "The Wikidata team" + } + ], + "description": "Structured data repository for MediaWiki", + "homepage": "http://wikiba.se";, + "keywords": [ + "wikibase", + "wikibaseclient", + "wikibaselib", + "wikibaserepo", + "wikidata" + ] } ] -- To view, visit https://gerrit.wikimedia.org/r/368312 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I01776073e8a9fbf440e4030f3493b209beef93cb Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/extensions/Wikidata Gerrit-Branch: wmf/1.30.0-wmf.10 Gerrit-Owner: Addshore <addshorew...@gmail.com> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
