Matthias Mullie has uploaded a new change for review. https://gerrit.wikimedia.org/r/179924
Change subject: [WIP] Take remaining logic out of UpdateOneSearchIndexConfig and into separate class ...................................................................... [WIP] Take remaining logic out of UpdateOneSearchIndexConfig and into separate class Change-Id: Id78c62cae779dddce4b2684e350d3d3173327956 --- A includes/Maintenance/ConfigUtils.php M maintenance/updateOneSearchIndexConfig.php 2 files changed, 196 insertions(+), 110 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/CirrusSearch refs/changes/24/179924/1 diff --git a/includes/Maintenance/ConfigUtils.php b/includes/Maintenance/ConfigUtils.php new file mode 100644 index 0000000..edc6722 --- /dev/null +++ b/includes/Maintenance/ConfigUtils.php @@ -0,0 +1,190 @@ +<?php + +namespace CirrusSearch\Maintenance; + +use Elastica\Client; +use Elastica\Index; +use Elastica\Query; +use Elastica\Type; + +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + */ +class ConfigUtils { // @todo: add to autoload + /** + * @var Client + */ + private $client; + + /** + * @var Maintenance + */ + private $out; + + /** + * @param Client $client + * @param Maintenance $out + */ + public function __construct( Client $client, Maintenance $out ) { + $this->client = $client; + $this->out = $out; + } + + public function checkElasticsearchVersion() { + $this->outputIndented( 'Fetching Elasticsearch version...' ); + $result = $this->client->request( '' ); + $result = $result->getData(); + if ( !isset( $result['version']['number'] ) ) { + $this->output( 'unable to determine, aborting.', 1 ); + } + $result = $result[ 'version' ][ 'number' ]; + $this->output( "$result..." ); + if ( !preg_match( '/^(1|2)./', $result ) ) { + $this->output( "Not supported!\n" ); + $this->error( "Only Elasticsearch 1.x is supported. Your version: $result.", 1 ); + } else { + $this->output( "ok\n" ); + } + } + + /** + * Pick the index identifier from the provided command line option. + * @param string $option command line option + * 'now' => current time + * 'current' => if there is just one index for this type then use its identifier + * other string => that string back + * @param string $typeName + * @return string index identifier to use + */ + public function pickIndexIdentifierFromOption( $option, $typeName ) { + if ( $option === 'now' ) { + $identifier = strval( time() ); + $this->outputIndented( "Setting index identifier...${typeName}_${identifier}\n" ); + return $identifier; + } + if ( $option === 'current' ) { + $this->outputIndented( 'Infering index identifier...' ); + $found = array(); + foreach ( $this->client->getStatus()->getIndexNames() as $name ) { + if ( substr( $name, 0, strlen( $typeName ) ) === $typeName ) { + $found[] = $name; + } + } + if ( count( $found ) > 1 ) { + $this->output( "error\n" ); + $this->error( "Looks like the index has more than one identifier. You should delete all\n" . + "but the one of them currently active. Here is the list: " . implode( $found, ',' ), 1 ); + } + if ( $found ) { + $identifier = substr( $found[0], strlen( $typeName ) + 1 ); + if ( !$identifier ) { + // This happens if there is an index named what the alias should be named. + // If the script is run with --startOver it should nuke it. + $identifier = 'first'; + } + } else { + $identifier = 'first'; + } + $this->output( "${typeName}_${identifier}\n "); + return $identifier; + } + return $option; + } + + /** + * @param array $bannedPlugins + * @return array + */ + public function scanAvailablePlugins( array $bannedPlugins = array() ) { + $this->outputIndented( "Scanning available plugins..." ); + $result = $this->client->request( '_nodes' ); + $result = $result->getData(); + $availablePlugins = array(); + $first = true; + foreach ( array_values( $result[ 'nodes' ] ) as $node ) { + $plugins = array(); + foreach ( $node[ 'plugins' ] as $plugin ) { + $plugins[] = $plugin[ 'name' ]; + } + if ( $first ) { + $availablePlugins = $plugins; + $first = false; + } else { + $availablePlugins = array_intersect( $availablePlugins, $plugins ); + } + } + if ( count( $availablePlugins ) === 0 ) { + $this->output( 'none' ); + } + $this->output( "\n" ); + if ( count( $bannedPlugins ) ) { + $availablePlugins = array_diff( $availablePlugins, $bannedPlugins ); + } + foreach ( array_chunk( $availablePlugins, 5 ) as $pluginChunk ) { + $plugins = implode( ', ', $pluginChunk ); + $this->outputIndented( "\t$plugins\n" ); + } + + return $availablePlugins; + } + + public function parsePotentialPercent( $str ) { + $result = floatval( $str ); + if ( strpos( $str, '%' ) === false ) { + return $result; + } + return $result / 100; + } + + // @todo: bring below options together in some abstract class where Validator & Reindexer also extend from + + /** + * @param string $message + * @param mixed $channel + */ + protected function output( $message, $channel = null ) { + if ( $this->out ) { + $this->out->output( $message, $channel ); + } + } + + /** + * @param string $message + */ + protected function outputIndented( $message ) { + if ( $this->out ) { + $this->out->outputIndented( $message ); + } + } + + /** + * @param string $message + * @param int $die + */ + private function error( $message, $die = 0 ) { + // @todo: I'll want to get rid of this method, but this patch will be big enough already + // @todo: I'll probably want to throw exceptions and/or return Status objects instead, later + + if ( $this->out ) { + $this->out->error( $message, $die ); + } + + $die = intval( $die ); + if ( $die > 0 ) { + die( $die ); + } + } +} diff --git a/maintenance/updateOneSearchIndexConfig.php b/maintenance/updateOneSearchIndexConfig.php index 2b8df13..2205996 100644 --- a/maintenance/updateOneSearchIndexConfig.php +++ b/maintenance/updateOneSearchIndexConfig.php @@ -178,12 +178,14 @@ // Set the timeout for maintenance actions $this->setConnectionTimeout(); + $utils = new ConfigUtils( $this->getClient(), $this ); + $this->indexType = $this->getOption( 'indexType' ); $this->startOver = $this->getOption( 'startOver', false ); $this->indexBaseName = $this->getOption( 'baseName', wfWikiId() ); $this->reindexAndRemoveOk = $this->getOption( 'reindexAndRemoveOk', false ); $this->reindexProcesses = $this->getOption( 'reindexProcesses', wfIsWindows() ? 1 : 5 ); - $this->reindexAcceptableCountDeviation = $this->parsePotentialPercent( + $this->reindexAcceptableCountDeviation = $utils->parsePotentialPercent( $this->getOption( 'reindexAcceptableCountDeviation', '5%' ) ); $this->reindexChunkSize = $this->getOption( 'reindexChunkSize', 100 ); $this->reindexRetryAttempts = $this->getOption( 'reindexRetryAttempts', 5 ); @@ -203,8 +205,8 @@ implode( ', ', $indexTypes ), 1 ); } - $this->checkElasticsearchVersion(); - $this->availablePlugins = $this->scanAvailablePlugins( $this->bannedPlugins ); + $utils->checkElasticsearchVersion(); + $this->availablePlugins = $utils->scanAvailablePlugins( $this->bannedPlugins ); if ( $this->getOption( 'justCacheWarmers', false ) ) { $this->validateCacheWarmers(); @@ -216,7 +218,7 @@ return; } - $this->indexIdentifier = $this->pickIndexIdentifierFromOption( $this->getOption( 'indexIdentifier', 'current' ), $this->getIndexTypeName() ); + $this->indexIdentifier = $utils->pickIndexIdentifierFromOption( $this->getOption( 'indexIdentifier', 'current' ), $this->getIndexTypeName() ); $this->analysisConfigBuilder = $this->pickAnalyzer( $this->langCode, $this->availablePlugins ); $this->validateIndex(); $this->validateAnalyzers(); @@ -239,60 +241,6 @@ "Message: $message\n" . "Trace:\n" . $trace, 1 ); } - } - - private function checkElasticsearchVersion() { - $this->outputIndented( 'Fetching Elasticsearch version...' ); - $result = $this->getClient()->request( '' ); - $result = $result->getData(); - if ( !isset( $result['version']['number'] ) ) { - $this->output( 'unable to determine, aborting.', 1 ); - } - $result = $result[ 'version' ][ 'number' ]; - $this->output( "$result..." ); - if ( !preg_match( '/^(1|2)./', $result ) ) { - $this->output( "Not supported!\n" ); - $this->error( "Only Elasticsearch 1.x is supported. Your version: $result.", 1 ); - } else { - $this->output( "ok\n" ); - } - } - - /** - * @param array $bannedPlugins - * @return array - */ - private function scanAvailablePlugins( array $bannedPlugins = array() ) { - $this->outputIndented( "Scanning available plugins..." ); - $result = $this->getClient()->request( '_nodes' ); - $result = $result->getData(); - $availablePlugins = array(); - $first = true; - foreach ( array_values( $result[ 'nodes' ] ) as $node ) { - $plugins = array(); - foreach ( $node[ 'plugins' ] as $plugin ) { - $plugins[] = $plugin[ 'name' ]; - } - if ( $first ) { - $availablePlugins = $plugins; - $first = false; - } else { - $availablePlugins = array_intersect( $availablePlugins, $plugins ); - } - } - if ( count( $availablePlugins ) === 0 ) { - $this->output( 'none' ); - } - $this->output( "\n" ); - if ( count( $bannedPlugins ) ) { - $availablePlugins = array_diff( $availablePlugins, $bannedPlugins ); - } - foreach ( array_chunk( $availablePlugins, 5 ) as $pluginChunk ) { - $plugins = implode( ', ', $pluginChunk ); - $this->outputIndented( "\t$plugins\n" ); - } - - return $availablePlugins; } private function updateVersions() { @@ -461,50 +409,6 @@ } /** - * Pick the index identifier from the provided command line option. - * @param string $option command line option - * 'now' => current time - * 'current' => if there is just one index for this type then use its identifier - * other string => that string back - * @param string $typeName - * @return string index identifier to use - */ - private function pickIndexIdentifierFromOption( $option, $typeName ) { - if ( $option === 'now' ) { - $identifier = strval( time() ); - $this->outputIndented( "Setting index identifier...${typeName}_${identifier}\n" ); - return $identifier; - } - if ( $option === 'current' ) { - $this->outputIndented( 'Infering index identifier...' ); - $found = array(); - foreach ( $this->getClient()->getStatus()->getIndexNames() as $name ) { - if ( substr( $name, 0, strlen( $typeName ) ) === $typeName ) { - $found[] = $name; - } - } - if ( count( $found ) > 1 ) { - $this->output( "error\n" ); - $this->error( "Looks like the index has more than one identifier. You should delete all\n" . - "but the one of them currently active. Here is the list: " . implode( $found, ',' ), 1 ); - } - if ( $found ) { - $identifier = substr( $found[0], strlen( $typeName ) + 1 ); - if ( !$identifier ) { - // This happens if there is an index named what the alias should be named. - // If the script is run with --startOver it should nuke it. - $identifier = 'first'; - } - } else { - $identifier = 'first'; - } - $this->output( "${typeName}_${identifier}\n "); - return $identifier; - } - return $option; - } - - /** * @param string $langCode * @param array $availablePlugins * @return AnalysisConfigBuilder @@ -632,14 +536,6 @@ // Otherwise its just a raw scalar so we should respect that too return $wgCirrusSearchReplicas; - } - - private function parsePotentialPercent( $str ) { - $result = floatval( $str ); - if ( strpos( $str, '%' ) === false ) { - return $result; - } - return $result / 100; } } -- To view, visit https://gerrit.wikimedia.org/r/179924 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: Id78c62cae779dddce4b2684e350d3d3173327956 Gerrit-PatchSet: 1 Gerrit-Project: mediawiki/extensions/CirrusSearch Gerrit-Branch: master Gerrit-Owner: Matthias Mullie <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
