Manybubbles has uploaded a new change for review.
https://gerrit.wikimedia.org/r/129571
Change subject: Use some analysis plugins if installed
......................................................................
Use some analysis plugins if installed
This adds support for Hebrew, Polish, Japanese, and Chinese plugins.
DEPLOYMENT: This does nothing if the plugins aren't installed and even
after that the index will have to be rebuilt to use the plugin. Once
it has been rebuilt, though, removing the plugin will break stuff.
Change-Id: Ic9a3e6b5168ce777df0dbd4a6839b2f2f11b4fa2
---
M includes/AnalysisConfigBuilder.php
M includes/MappingConfigBuilder.php
M maintenance/updateOneSearchIndexConfig.php
3 files changed, 48 insertions(+), 9 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/CirrusSearch
refs/changes/71/129571/1
diff --git a/includes/AnalysisConfigBuilder.php
b/includes/AnalysisConfigBuilder.php
index 2d744f8..8e7f101 100644
--- a/includes/AnalysisConfigBuilder.php
+++ b/includes/AnalysisConfigBuilder.php
@@ -29,7 +29,7 @@
* and change the minor version when it changes but isn't
* incompatible
*/
- const VERSION = '0.5';
+ const VERSION = '0.6';
/**
* Language code we're building analysis for
@@ -40,9 +40,15 @@
/**
* Constructor
* @param string $langCode The language code to build config for
+ * @param array(string) $plugins list of plugins installed in
Elasticsearch
*/
- public function __construct( $langCode ) {
+ public function __construct( $langCode, $plugins ) {
$this->language = $langCode;
+ foreach ( $this->elasticsearchLanguageAnalyzersFromPlugins as
$plugin => $extra ) {
+ if ( in_array( $plugin, $plugins ) ) {
+ $this->elasticsearchLanguageAnalyzers =
array_merge( $this->elasticsearchLanguageAnalyzers, $extra );
+ }
+ }
}
/**
@@ -62,6 +68,9 @@
return array(
'analyzer' => array(
'text' => array(
+ 'type' =>
$this->getDefaultTextAnalyzerType(),
+ ),
+ 'text_search' => array(
'type' =>
$this->getDefaultTextAnalyzerType(),
),
'plain' => array(
@@ -193,10 +202,19 @@
$config[ 'analyzer' ][ 'prefix' ][ 'filter' ][] =
'asciifolding';
$config[ 'analyzer' ][ 'lowercase_keyword' ][ 'filter'
][] = 'asciifolding';
$config[ 'analyzer' ][ 'near_match' ][ 'filter' ][] =
'asciifolding';
+
+ // In English text_search is just a copy of text
+ $config[ 'analyzer' ][ 'text_search' ] = $config[
'analyzer' ][ 'text' ];
break;
case 'tr':
$config[ 'filter' ][ 'lowercase' ][ 'language' ] =
'turkish';
break;
+ case 'he':
+ // If the hebrew plugin kicked us over to the hebrew
analyzer use its companion
+ // analyzer for queries.
+ if ( $config[ 'analyzer' ][ 'text_search' ] ===
'hebrew' ) {
+ $config[ 'analyzer' ][ 'text_search' ] =
'hebrew_query';
+ }
}
return $config;
}
@@ -207,7 +225,7 @@
* reasonably default in case CirrusSearch isn't customized for the
language.
* @return string the analyzer type
*/
- private function getDefaultTextAnalyzerType() {
+ public function getDefaultTextAnalyzerType() {
// If we match a language exactly, use it
if ( array_key_exists( $this->language,
$this->elasticsearchLanguageAnalyzers ) ) {
return $this->elasticsearchLanguageAnalyzers[
$this->language ];
@@ -264,4 +282,14 @@
'tr' => 'turkish',
'th' => 'thai',
);
+
+ private $elasticsearchLanguageAnalyzersFromPlugins = array(
+ 'analysis-stempel' => array( 'pl' => 'polish' ),
+ 'analysis-kuromoji' => array( 'ja' => 'kuromoji' ),
+ 'analysis-smartcn' => array( 'zh-hans' => 'smart_chinese' ),
+ // This hasn't had a release in a while and seems to not work
with the
+ // current version of elasticsearch:
+ 'elasticsearch-analysis-hebrew' => array( 'he' => 'hebrew' ),
+ // TODO Hebrew requires some special query handling....
+ );
}
diff --git a/includes/MappingConfigBuilder.php
b/includes/MappingConfigBuilder.php
index 9751d69..55338cf 100644
--- a/includes/MappingConfigBuilder.php
+++ b/includes/MappingConfigBuilder.php
@@ -158,12 +158,13 @@
// multi_field is dead in 1.0 so we do this which actually
looks less gnarly.
$field = array(
'type' => 'string',
- 'analyzer' => 'text',
+ 'index_analyzer' => 'text',
+ 'search_analyzer' => 'text_search',
'fields' => array(
'plain' => array(
'type' => 'string',
'index_analyzer' => 'plain',
- 'search_analyzer' => 'plain_search'
+ 'search_analyzer' => 'plain_search',
),
)
);
diff --git a/maintenance/updateOneSearchIndexConfig.php
b/maintenance/updateOneSearchIndexConfig.php
index 1b28016..98d50fc 100644
--- a/maintenance/updateOneSearchIndexConfig.php
+++ b/maintenance/updateOneSearchIndexConfig.php
@@ -94,6 +94,11 @@
private $reindexAcceptableCountDeviation;
/**
+ * @var the builder for analysis config
+ */
+ private $analysisConfigBuilder;
+
+ /**
* @var array(String) list of available plugins
*/
private $availablePlugins;
@@ -194,6 +199,7 @@
}
$this->indexIdentifier =
$this->pickIndexIdentifierFromOption( $this->getOption( 'indexIdentifier',
'current' ) );
+ $this->pickAnalyzer();
$this->validateIndex();
$this->validateAnalyzers();
$this->validateMapping();
@@ -313,8 +319,7 @@
private function validateAnalyzers() {
$this->output( $this->indent . "Validating analyzers..." );
$settings = $this->getSettings();
- $analysisConfig = new AnalysisConfigBuilder( $this->langCode );
- $requiredAnalyzers = $analysisConfig->buildConfig();
+ $requiredAnalyzers =
$this->analysisConfigBuilder->buildConfig();
if ( $this->checkConfig( $settings[ 'analysis' ],
$requiredAnalyzers ) ) {
$this->output( "ok\n" );
} else {
@@ -823,12 +828,11 @@
private function createIndex( $rebuild ) {
global $wgCirrusSearchRefreshInterval;
- $analysisConfig = new AnalysisConfigBuilder( $this->langCode );
$this->getIndex()->create( array(
'settings' => array(
'number_of_shards' => $this->getShardCount(),
'number_of_replicas' =>
$this->reindexAndRemoveOk ? 0 : $this->getReplicaCount(),
- 'analysis' => $analysisConfig->buildConfig(),
+ 'analysis' =>
$this->analysisConfigBuilder->buildConfig(),
'translog.flush_threshold_ops' => 50000, //
This is supposed to help with bulk index io load.
'index.query.default_field' => 'page.text', //
Since the _all field is disabled, we should query something.
'refresh_interval' =>
$wgCirrusSearchRefreshInterval . 's'
@@ -882,6 +886,12 @@
return $option;
}
+ private function pickAnalyzer() {
+ $this->analysisConfigBuilder = new AnalysisConfigBuilder(
$this->langCode, $this->availablePlugins );
+ $this->output( $this->indent . 'Picking analyzer...' .
+
$this->analysisConfigBuilder->getDefaultTextAnalyzerType() . "\n" );
+ }
+
/**
* @return \Elastica\Index being updated
*/
--
To view, visit https://gerrit.wikimedia.org/r/129571
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ic9a3e6b5168ce777df0dbd4a6839b2f2f11b4fa2
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/CirrusSearch
Gerrit-Branch: master
Gerrit-Owner: Manybubbles <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
