DCausse has uploaded a new change for review.
https://gerrit.wikimedia.org/r/258131
Change subject: Makes suggester index creation more robust
......................................................................
Makes suggester index creation more robust
Changed the two pass creation (createIndex, putMapping) into a onepass
creation.
Removed analyzer validation (useless analysis config cannot change).
Added a check to wait for the index to be ready before indexing.
Change-Id: Ic66d17b320f48e5d39729b11372204cf89fd6d3a
---
M includes/ClusterSettings.php
M includes/Maintenance/ConfigUtils.php
M maintenance/updateSuggesterIndex.php
3 files changed, 58 insertions(+), 31 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/CirrusSearch
refs/changes/31/258131/1
diff --git a/includes/ClusterSettings.php b/includes/ClusterSettings.php
index 868bdaa..bcddb40 100644
--- a/includes/ClusterSettings.php
+++ b/includes/ClusterSettings.php
@@ -39,7 +39,7 @@
return $settings[$indexType];
}
throw new \Exception( "Could not find a shard count for "
- . "{$this->indexType}. Did you add an index to "
+ . "{$indexType}. Did you add an index to "
. "\$wgCirrusSearchNamespaceMappings but forget to "
. "add it to \$wgCirrusSearchShardCount?" );
}
diff --git a/includes/Maintenance/ConfigUtils.php
b/includes/Maintenance/ConfigUtils.php
index 7d16c5d..4ffe4ba 100644
--- a/includes/Maintenance/ConfigUtils.php
+++ b/includes/Maintenance/ConfigUtils.php
@@ -176,4 +176,43 @@
die( $die );
}
}
+
+ /**
+ * Get index health
+ * @param string $indexName
+ * @return array the index health status
+ */
+ public function getIndexHealth( $indexName ) {
+ $path = "_cluster/health/$indexName";
+ $response = $this->client->request( $path );
+ if ( $response->hasError() ) {
+ throw new \Exception( "Error while fetching index
health status: ". $response->getError() );
+ }
+ return $response->getData();
+ }
+
+ /**
+ * Wait for the index to go green
+ * @param string $indexName
+ * @param int $timeout
+ * @return boolean true if the index is green false otherwise.
+ */
+ public function waitForGreen( $indexName, $timeout ) {
+ $startTime = time();
+ while( ( $startTime + $timeout ) > time() ) {
+ try {
+ $response = $this->getIndexHealth( $indexName );
+ $status = isset ( $response['status'] ) ?
$response['status'] : 'unknown';
+ if ( $status == 'green' ) {
+ $this->outputIndented( "\tGreen!\n" );
+ return true;
+ }
+ $this->outputIndented( "\tIndex is $status
retrying...\n" );
+ sleep( 5 );
+ } catch( \Exception $e ) {
+ $this->output( "Error while waiting for green
({$e->getMessage()}), retrying...\n" );
+ }
+ }
+ return false;
+ }
}
diff --git a/maintenance/updateSuggesterIndex.php
b/maintenance/updateSuggesterIndex.php
index fcee0cb..e7357a9 100644
--- a/maintenance/updateSuggesterIndex.php
+++ b/maintenance/updateSuggesterIndex.php
@@ -124,6 +124,14 @@
$this->masterTimeout = $this->getOption( 'masterTimeout',
$wgCirrusSearchMasterTimeout );
$this->indexTypeName = Connection::TITLE_SUGGEST_TYPE;
+ // Check that all shards and replicas settings are set
+ try {
+ $this->getShardCount();
+ $this->getReplicaCount();
+ } catch( \Exception $e ) {
+ $this->error( "Failed to get shard count and replica
count information: {$e->getMessage()}", 1 );
+ }
+
// Make sure we don't flood the pool counter
unset( $wgPoolCounterConf['CirrusSearch-Search'] );
@@ -156,8 +164,11 @@
$this->analysisConfigBuilder = $this->pickAnalyzer(
$this->langCode, $this->availablePlugins );
$this->createIndex();
- $this->validateAnalyzers();
- $this->createMapping();
+ $this->output( "Waiting for the index to go green...\n"
);
+ // Wait for the index to go green
+ if ( !$utils->waitForGreen(
$this->getIndex()->getName(), 600 ) ) {
+ $this->error( "Failed to wait for green...
please check config and delete the {$this->getIndex()->getName()} index if it
was created." );
+ }
$this->indexData();
if ( $this->optimizeIndex ) {
$this->optimize();
@@ -325,13 +336,17 @@
throw new \Exception( "Index already exists." );
}
+ $type = $this->getType();
+ $mappingConfigBuilder = new SuggesterMappingConfigBuilder();
+
$args = array(
'settings' => array(
'number_of_shards' => $this->getShardCount(),
'auto_expand_replicas' =>
$this->getReplicaCount(),
'analysis' =>
$this->analysisConfigBuilder->buildConfig(),
'routing.allocation.total_shards_per_node' =>
$maxShardsPerNode,
- )
+ ),
+ 'mappings' => $mappingConfigBuilder->buildConfig()
);
// @todo utilize $this->getIndex()->create(...) once it
supports setting
// the master_timeout parameter.
@@ -346,25 +361,6 @@
private function getReplicaCount() {
return $this->getConnection()->getSettings()->getReplicaCount(
$this->indexTypeName );
- }
-
- private function createMapping() {
- $type = $this->getType();
- $mappingConfigBuilder = new SuggesterMappingConfigBuilder();
- $validator = new
\CirrusSearch\Maintenance\Validators\MappingValidator(
- $this->getIndex(),
- $this->masterTimeout,
- false,
- $this->availablePlugins,
- $mappingConfigBuilder->buildConfig(),
- array( Connection::TITLE_SUGGEST_TYPE_NAME => $type ),
- $this
- );
-
- $status = $validator->validate();
- if ( !$status->isOK() ) {
- $this->error( $status->getMessage()->text(), 1 );
- }
}
private function getShardCount() {
@@ -403,14 +399,6 @@
$validators[] = $this->getShardAllocationValidator();
$validators[] = new
\CirrusSearch\Maintenance\Validators\MaxShardsPerNodeValidator(
$this->getIndex(), $this->indexTypeName, $this->maxShardsPerNode, $this );
return $validators;
- }
-
- private function validateAnalyzers() {
- $validator = new
\CirrusSearch\Maintenance\Validators\AnalyzersValidator( $this->getIndex(),
$this->analysisConfigBuilder, $this );
- $status = $validator->validate();
- if ( !$status->isOK() ) {
- $this->error( $status->getMessage()->text(), 1 );
- }
}
/**
--
To view, visit https://gerrit.wikimedia.org/r/258131
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ic66d17b320f48e5d39729b11372204cf89fd6d3a
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/CirrusSearch
Gerrit-Branch: master
Gerrit-Owner: DCausse <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
