Manybubbles has uploaded a new change for review.
https://gerrit.wikimedia.org/r/89104
Change subject: Let forceSearchUpdate skip pages that are current.
......................................................................
Let forceSearchUpdate skip pages that are current.
This comes at surprisingly steep performance cost if the page is very
cheep to render. Since most pages aren't that cheap, this is worth it.
Change-Id: I6b8899f36d1d5ec2df9bf37882978c19ee095578
---
M includes/CirrusSearchSearcher.php
M includes/CirrusSearchUpdater.php
M maintenance/forceSearchIndex.php
3 files changed, 73 insertions(+), 35 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/CirrusSearch
refs/changes/04/89104/1
diff --git a/includes/CirrusSearchSearcher.php
b/includes/CirrusSearchSearcher.php
index 4d36966..9d2c7ba 100644
--- a/includes/CirrusSearchSearcher.php
+++ b/includes/CirrusSearchSearcher.php
@@ -208,38 +208,19 @@
// It'd be better to be able to have Elasticsearch fetch this
during the query rather than make
// two passes but it doesn't support that at this point
- $indexType = $this->pickIndexTypeFromNamespaces();
- $getWork = new PoolCounterWorkViaCallback(
'CirrusSearch-Search', "_elasticsearch", array(
- 'doWork' => function() use ( $id, $indexType ) {
- try {
- $result =
CirrusSearchConnection::getPageType( $indexType )->getDocument( $id, array(
- 'fields' => array( 'text' ),
- ) );
- return $result;
- } catch ( \Elastica\Exception\NotFoundException
$e ) {
- // We don't need to log
NotFoundExceptions....
- return null;
- } catch (
\Elastica\Exception\ExceptionInterface $e ) {
- wfLogWarning( "Search backend error
during get for $id. Error message is: " . $e->getMessage() );
- return false;
- }
- }
- ) );
- $getResult = $getWork->execute();
- if ( $getResult === null ) {
- // This corresponds to not found exceptions
- return null;
+ $found = $this->get( $id, array( 'text' ) );
+ if ( !$found->isOk() ) {
+ return $found;
}
- if ( $getResult === false ) {
- // These are actual errors
- $status = new Status();
- $status->warning( 'cirrussearch-backend-error' );
- return $status;
+ $found = $found->getValue();
+ if ( $found === null ) {
+ // If the pge doesn't exist we can't find any articles
like it
+ return null;
}
$this->query = new \Elastica\Query\MoreLikeThis();
$this->query->setParams( $wgCirrusSearchMoreLikeThisConfig );
- $this->query->setLikeText( Sanitizer::stripAllTags(
$getResult->text ) );
+ $this->query->setLikeText( Sanitizer::stripAllTags(
$found->text ) );
$this->query->setFields( array( 'text' ) );
$idFilter = new \Elastica\Filter\Ids();
$idFilter->addId( $id );
@@ -249,6 +230,36 @@
}
/**
+ * Get the page with $id.
+ * @param $id int page id
+ * @param $fields array(string) fields to fetch
+ * @return Status containing page data, null if not found, or a if
there was an error
+ */
+ public function get( $id, $fields ) {
+ $indexType = $this->pickIndexTypeFromNamespaces();
+ $getWork = new PoolCounterWorkViaCallback(
'CirrusSearch-Search', "_elasticsearch", array(
+ 'doWork' => function() use ( $indexType, $id, $fields )
{
+ try {
+ $result =
CirrusSearchConnection::getPageType( $indexType )->getDocument( $id, array(
+ 'fields' => $fields,
+ ) );
+ return Status::newGood( $result );
+ } catch ( \Elastica\Exception\NotFoundException
$e ) {
+ // NotFoundException just means the
field didn't exist.
+ // It is up to the called to decide if
that is and error.
+ return Status::newGood( null );
+ } catch (
\Elastica\Exception\ExceptionInterface $e ) {
+ wfLogWarning( "Search backend error
during get for $id. Error message is: " . $e->getMessage() );
+ $status = new Status();
+ $status->warning(
'cirrussearch-backend-error' );
+ return $status;
+ }
+ }
+ ) );
+ return $getWork->execute();
+ }
+
+ /**
* Powers full-text-like searches which means pretty much everything
but prefixSearch.
* @return CirrusSearchResultSet|null|SearchResultSet|Status
*/
diff --git a/includes/CirrusSearchUpdater.php b/includes/CirrusSearchUpdater.php
index d778045..0127f22 100644
--- a/includes/CirrusSearchUpdater.php
+++ b/includes/CirrusSearchUpdater.php
@@ -116,12 +116,15 @@
* )
* )
*/
- public static function updateRevisions( $pageData ) {
+ public static function updateRevisions( $pageData, $checkFreshness =
false ) {
wfProfileIn( __METHOD__ );
$contentDocuments = array();
$generalDocuments = array();
foreach ( $pageData as $page ) {
+ if ( $checkFreshness && self::isFresh( $page ) ) {
+ continue;
+ }
$document = self::buildDocumentforRevision( $page );
if ( MWNamespace::isContent( $document->get(
'namespace' ) ) ) {
$contentDocuments[] = $document;
@@ -132,7 +135,9 @@
self::sendDocuments(
CirrusSearchConnection::CONTENT_INDEX_TYPE, $contentDocuments );
self::sendDocuments(
CirrusSearchConnection::GENERAL_INDEX_TYPE, $generalDocuments );
+ $count = count( $contentDocuments ) + count( $generalDocuments
);
wfProfileOut( __METHOD__ );
+ return $count;
}
/**
@@ -149,6 +154,25 @@
wfProfileOut( __METHOD__ );
}
+ private static function isFresh( $page ) {
+ $page = $page[ 'page' ];
+ $searcher = new CirrusSearchSearcher( 0, 0, array(
$page->getTitle()->getNamespace() ) );
+ $get = $searcher->get( $page->getTitle()->getArticleId(),
array( 'timestamp ') );
+ if ( !$get->isOk() ) {
+ return false;
+ }
+ $get = $get->getValue();
+ if ( $get === null ) {
+ return false;
+ }
+ $found = new MWTimestamp( $get->timestamp );
+ $diff = $found->diff( new MWTimestamp( $page->getTimestamp() )
);
+ if ( $diff === false ) {
+ return false;
+ }
+ return !$diff->invert;
+ }
+
/**
* @param $indexType
* @param $documents array
diff --git a/maintenance/forceSearchIndex.php b/maintenance/forceSearchIndex.php
index a36cda3..96bf61a 100644
--- a/maintenance/forceSearchIndex.php
+++ b/maintenance/forceSearchIndex.php
@@ -34,6 +34,7 @@
var $toId = null;
var $indexUpdates;
var $limit;
+ var $forceUpdate;
public function __construct() {
parent::__construct();
@@ -50,6 +51,8 @@
$this->addOption( 'limit', 'Maximum number of pages to process
before exiting the script. Default to unlimited.', false, true );
$this->addOption( 'buildChunks', 'Instead of running the script
spit out N commands that can be farmed out to ' .
'different processes or machines to rebuild the index.
Works with fromId and toId, not from and to.', false, true );
+ $this->addOption( 'forceUpdate', 'Blindly upload pages to
Elasticsearch whether or not it already has an up ' .
+ 'to date copy. Not used with --deletes.' );
}
public function execute() {
@@ -67,6 +70,7 @@
$this->buildChunks( $buildChunks );
return;
}
+ $this->forceUpdate = $this->getOption( 'forceUpdate' );
if ( $this->indexUpdates ) {
$operationName = 'Indexed';
@@ -108,18 +112,17 @@
$revisions = array_filter($revisions, function
($rev) {
return $rev[ 'page' ] !== null;
});
- // Update size to reflect stripped entries.
- $size = count( $revisions );
- CirrusSearchUpdater::updateRevisions(
$revisions );
+ // Update size with the actual number of
updated documents.
+ $size = CirrusSearchUpdater::updateRevisions(
$revisions, !$this->forceUpdate );
} else {
$idsToDelete = array();
foreach( $titles as $t ) {
- $idsToDelete[] = $t['page'];
+ $idsToDelete[] = $t[ 'page' ];
$lastTitle = $t;
}
- $minUpdate = $lastTitle['timestamp'];
- $minNamespace = $lastTitle['namespace'];
- $minTitle = $lastTitle['title'];
+ $minUpdate = $lastTitle[ 'timestamp' ];
+ $minNamespace = $lastTitle[ 'namespace' ];
+ $minTitle = $lastTitle[ 'title' ];
CirrusSearchUpdater::deletePages( $idsToDelete
);
}
$completed += $size;
--
To view, visit https://gerrit.wikimedia.org/r/89104
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I6b8899f36d1d5ec2df9bf37882978c19ee095578
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/CirrusSearch
Gerrit-Branch: master
Gerrit-Owner: Manybubbles <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
