EBernhardson has uploaded a new change for review.
https://gerrit.wikimedia.org/r/281262
Change subject: WIP index archived titles
......................................................................
WIP index archived titles
Change-Id: I25b2ee0f4dd5a28da5087821511d0caead6aa63b
---
M includes/DataSender.php
M includes/Hooks.php
M includes/Job/DeletePages.php
M includes/Updater.php
M maintenance/forceSearchIndex.php
5 files changed, 121 insertions(+), 4 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/CirrusSearch
refs/changes/62/281262/1
diff --git a/includes/DataSender.php b/includes/DataSender.php
index ad344d3..63b8f17 100644
--- a/includes/DataSender.php
+++ b/includes/DataSender.php
@@ -243,6 +243,33 @@
return Status::newGood();
}
+ public function sendArchiveUpdates( array $archived ) {
+ $index = $this->connection->getPageType( 'archive' );
+ if ( !$this->areIndexesAvailableForWrites( array( 'archive' ),
true ) ) {
+ return Status::newFatal( 'cirrussearch-indexes-frozen'
);
+ }
+
+ try {
+ $this->start( "indexing {numIds} archived pages", array(
+ 'numIds' => count( $archived ),
+ 'queryType' => 'send_archive_updates',
+ ) );
+ $bulk = new \Elastica\Bulk(
$this->connection->getClient() );
+ $bulk->setType( $index );
+ $bulk->addData( $archived, 'update' );
+ $bulk->send();
+ } catch ( \Elastica\Exception\ExceptionInterface $e ) {
+ $this->failure( $d );
+ $this->failedLog->warning(
+ '@todo a better error message',
+ array( 'exception' => $e )
+ );
+ return Status::newFatal(
'cirrussearch-failed-archive-updates' );
+ }
+
+ return Status::newGood();
+ }
+
/**
* @param string $localSite The wikiId to add/remove from
local_sites_with_dupe
* @param string $indexName The name of the index to perform updates to
diff --git a/includes/Hooks.php b/includes/Hooks.php
index 7b0377f..40cb325 100644
--- a/includes/Hooks.php
+++ b/includes/Hooks.php
@@ -293,7 +293,11 @@
// Note that we must use the article id provided or it'll be
lost in the ether. The job can't
// load it from the title because the page row has already been
deleted.
JobQueueGroup::singleton()->push(
- new Job\DeletePages( $page->getTitle(), array( 'id' =>
$pageId ) )
+ new Job\DeletePages( $page->getTitle(), array(
+ 'id' => $pageId,
+ 'timestamp' => wfTimestampNow(),
+ 'archive' => true,
+ ) )
);
return true;
}
diff --git a/includes/Job/DeletePages.php b/includes/Job/DeletePages.php
index d794977..2cc1c97 100644
--- a/includes/Job/DeletePages.php
+++ b/includes/Job/DeletePages.php
@@ -36,7 +36,23 @@
$updater = $this->createUpdater();
$indexType = isset( $this->params[ 'indexType' ] ) ?
$this->params[ 'indexType' ] : null;
- return $updater->deletePages( array( $this->title ), array(
$this->params[ 'id' ] ),
- $wgCirrusSearchClientSideUpdateTimeout, $indexType );
+ $success = $updater->deletePages(
+ $wgCirrusSearchClientSideUpdateTimeout, $indexType
+ );
+
+ if ( $success && !empty( $this->params['archive'] ) ) {
+ $success = $updater->archivePages(
+ array(
+ array(
+ 'title' => $this->title,
+ 'page' => $this->params['id'],
+ 'timestamp' =>
$this->params['timestamp']
+ ),
+ ),
+ $wgCirrusSearchClientSideUpdateTimeout
+ );
+ }
+
+ return $success;
}
}
diff --git a/includes/Updater.php b/includes/Updater.php
index 38d19cf..597bc74 100644
--- a/includes/Updater.php
+++ b/includes/Updater.php
@@ -260,9 +260,67 @@
// This job type will insert itself into the job queue
// with a delay if writes to ES are currently paused
$job->run();
+
+ return true;
}
/**
+ * @param Title $title
+ * @param array $archived
+ * @param string $clientSideTimeout
+ * @return bool
+ */
+ public function archivePages( $archived, $clientSideTimeout = null ) {
+ $docs = $this->buildArchiveDocuments( $archived );
+ $head = reset( $archived );
+ foreach ( array_chunk( $docs, 10 ) as $chunked ) {
+ $job = new Job\ElasticaWrite(
+ $head['title'],
+ array(
+ 'clientSideTimeout' =>
$clientSideTimeout,
+ 'method' => 'sendArchiveUpdates',
+ 'arguments' => array( $chunked ),
+ 'cluster' => $this->writeToClusterName
+ )
+ );
+ $job->run();
+ }
+
+ return true;
+ }
+
+ private function buildArchiveDocuments( array $archived ) {
+ global $wgCirrusSearchUpdateConflictRetryCount;
+
+ foreach ( $archived as $delete ) {
+ if ( !isset( $delete['title'] ) ) {
+ // These come from pages that still exist, but
are redirects.
+ // This is non-obvious and we probably need a
better way...
+ continue;
+ }
+ // We need some reliable id that can be used to ensure
we
+ // don't do duplicate updates, but that also will not
have
+ // conflicts cross-wiki.
+ $id = implode( ':', array(
+ wfWikiId(),
+ $delete['title']->getNamespace(),
+ $delete['title']->getText(),
+ ) );
+ $id = base64_encode( md5( $id, true ) );
+ $doc = new \Elastica\Document( $id, array(
+ 'namespace' => $delete['title']->getNamespace(),
+ 'title' => $delete['title']->getText(),
+ 'timestamp' => wfTimestamp( TS_ISO_8601,
$delete['timestamp'] )
+ ) );
+ $doc->setDocAsUpsert( true );
+ $doc->setRetryOnConflict(
$wgCirrusSearchUpdateConflictRetryCount );
+
+ $docs[] = $doc;
+ }
+
+ return $docs;
+ }
+ /**
* @param \WikiPage[] $pages
* @param int $flags
* @return \Elastica\Document[]
diff --git a/maintenance/forceSearchIndex.php b/maintenance/forceSearchIndex.php
index ac6af80..277c1fd 100644
--- a/maintenance/forceSearchIndex.php
+++ b/maintenance/forceSearchIndex.php
@@ -44,6 +44,7 @@
public $toDate = null;
public $toId = null;
public $indexUpdates;
+ public $archiveOnly;
public $limit;
public $queue;
public $maxJobs;
@@ -64,6 +65,7 @@
$this->addOption( 'fromId', 'Start indexing at a specific
page_id. Not useful with --deletes.', false, true );
$this->addOption( 'toId', 'Stop indexing at a specific page_id.
Not useful with --deletes or --from or --to.', false, true );
$this->addOption( 'deletes', 'If this is set then just index
deletes, not updates or creates.', false );
+ $this->addOption( 'archiveOnly', 'Don\'t delete pages, only
index them into the archive. Only useful with --deletes', false, true );
$this->addOption( 'limit', 'Maximum number of pages to process
before exiting the script. Default to unlimited.', false, true );
$this->addOption( 'buildChunks', 'Instead of running the script
spit out commands that can be farmed out to ' .
'different processes or machines to rebuild the index.
Works with fromId and toId, not from and to. ' .
@@ -113,6 +115,7 @@
}
$this->toId = $this->getOption( 'toId' );
$this->indexUpdates = !$this->getOption( 'deletes', false );
+ $this->archiveOnly = (bool) $this->getOption( 'archiveOnly',
true );
$this->limit = $this->getOption( 'limit' );
$buildChunks = $this->getOption( 'buildChunks' );
if ( $buildChunks !== null ) {
@@ -224,16 +227,25 @@
} else {
$titlesToDelete = array();
$idsToDelete = array();
+ $archive = array();
foreach( $deletes as $delete ) {
$titlesToDelete[] = $delete[ 'title' ];
$idsToDelete[] = $delete[ 'page' ];
$lastDelete = $delete;
+ $archive[] = array(
+ 'title' => $delete['title'],
+ 'page' => $delete['page'],
+ 'timestamp' =>
$delete['timestamp'],
+ );
}
$minUpdate = $lastDelete[ 'timestamp' ];
$minNamespace = $lastDelete[ 'title'
]->getNamespace();
$minTitle = $lastDelete[ 'title' ]->getText();
$updater = $this->createUpdater();
- $updater->deletePages( $titlesToDelete,
$idsToDelete );
+ $updater->archivePages( $archive );
+ if ( !$this->archiveOnly ) {
+ $updater->deletePages( $titlesToDelete,
$idsToDelete );
+ }
}
$completed += $size;
--
To view, visit https://gerrit.wikimedia.org/r/281262
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I25b2ee0f4dd5a28da5087821511d0caead6aa63b
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/CirrusSearch
Gerrit-Branch: master
Gerrit-Owner: EBernhardson <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
