jenkins-bot has submitted this change and it was merged.
Change subject: Added Similarity configuration
......................................................................
Added Similarity configuration
* User can now configure the similarity function and parameters used by the
index
* Re-enabled norms on arrays fields
Norms were explicity disabled on array fields which maybe makes sense with the
default similiraty, docs with a high number of redirects were maybe ranked too
low
but without norms it's the opposite, used with a DisMax the redirect field will
always win thus docs with a high number of redirects are likely to win.
I think it's better to re-enable them and allow the user to fine tune a
parameter
that affects norms (setting a very low b value with bm25).
Bug: T128061
Change-Id: Ie718c8a03ea5351e2e142f22436e2fd83314056f
---
M CirrusSearch.php
M includes/Maintenance/AnalysisConfigBuilder.php
M includes/Maintenance/IndexCreator.php
M includes/Maintenance/MappingConfigBuilder.php
A profiles/SimilarityProfiles.php
5 files changed, 166 insertions(+), 19 deletions(-)
Approvals:
Cindy-the-browser-test-bot: Looks good to me, but someone else must approve
EBernhardson: Looks good to me, approved
jenkins-bot: Verified
diff --git a/CirrusSearch.php b/CirrusSearch.php
index 78e87b9..33e6417 100644
--- a/CirrusSearch.php
+++ b/CirrusSearch.php
@@ -25,6 +25,7 @@
require_once __DIR__ . "/profiles/PhraseSuggesterProfiles.php";
require_once __DIR__ . "/profiles/CommonTermsQueryProfiles.php";
require_once __DIR__ . "/profiles/RescoreProfiles.php";
+require_once __DIR__ . "/profiles/SimilarityProfiles.php";
$wgExtensionCredits['other'][] = array(
'path' => __FILE__,
@@ -344,6 +345,10 @@
// Maximum number of newly unlinked articles to update when an article changes.
$wgCirrusSearchUnlinkedArticlesToUpdate = 25;
+// Configure the similarity module
+// see profile/SimilarityProfiles.php for more details
+$wgCirrusSearchSimilarityProfile =
$wgCirrusSearchSimilarityProfiles['default'];
+
// Weight of fields. Must be integers not decimals. If
$wgCirrusSearchAllFields['use']
// is false this can be changed on the fly. If it is true then changes to
this require
// an in place reindex to take effect.
diff --git a/includes/Maintenance/AnalysisConfigBuilder.php
b/includes/Maintenance/AnalysisConfigBuilder.php
index 3e29209..5ba02d1 100644
--- a/includes/Maintenance/AnalysisConfigBuilder.php
+++ b/includes/Maintenance/AnalysisConfigBuilder.php
@@ -2,6 +2,8 @@
namespace CirrusSearch\Maintenance;
+use ConfigFactory;
+use \CirrusSearch\SearchConfig;
use \CirrusSearch\Searcher;
use \Hooks;
use \Language;
@@ -44,12 +46,15 @@
*/
private $icu;
+ private $similarity;
+
/**
* Constructor
* @param string $langCode The language code to build config for
* @param array(string) $plugins list of plugins installed in
Elasticsearch
+ * @param SearchConfig $config
*/
- public function __construct( $langCode, $plugins ) {
+ public function __construct( $langCode, $plugins, $config = null ) {
$this->language = $langCode;
foreach ( $this->elasticsearchLanguageAnalyzersFromPlugins as
$plugin => $extra ) {
if ( in_array( $plugin, $plugins ) ) {
@@ -57,6 +62,10 @@
}
}
$this->icu = in_array( 'analysis-icu', $plugins );
+ if ( is_null ( $config ) ) {
+ $config =
ConfigFactory::getDefaultInstance()->makeConfig( 'CirrusSearch' );
+ }
+ $this->similarity = $config->get(
'CirrusSearchSimilarityProfile' );
}
/**
@@ -67,6 +76,16 @@
$config = $this->customize( $this->defaults() );
Hooks::run( 'CirrusSearchAnalysisConfig', array( &$config ) );
return $config;
+ }
+
+ /**
+ * Build the similarity config
+ * @return array the similarity config
+ */
+ public function buildSimilarityConfig() {
+ if ( $this->similarity != null && isset (
$this->similarity['similarity'] ) ) {
+ return $this->similarity['similarity'];
+ }
}
/**
@@ -233,6 +252,7 @@
'name' => 'nfkc_cf',
);
}
+
return $defaults;
}
diff --git a/includes/Maintenance/IndexCreator.php
b/includes/Maintenance/IndexCreator.php
index 93eecf2..518ea52 100644
--- a/includes/Maintenance/IndexCreator.php
+++ b/includes/Maintenance/IndexCreator.php
@@ -96,6 +96,10 @@
'routing.allocation.total_shards_per_node' =>
$maxShardsPerNode,
)
);
+ $similarity =
$this->analysisConfigBuilder->buildSimilarityConfig();
+ if ( $similarity ) {
+ $args['settings']['similarity'] = $similarity;
+ }
if ( $searchAllFields ) {
// Use our weighted all field as the default rather
than _all which is disabled.
diff --git a/includes/Maintenance/MappingConfigBuilder.php
b/includes/Maintenance/MappingConfigBuilder.php
index f512e38..4767823 100644
--- a/includes/Maintenance/MappingConfigBuilder.php
+++ b/includes/Maintenance/MappingConfigBuilder.php
@@ -1,6 +1,8 @@
<?php
namespace CirrusSearch\Maintenance;
+use ConfigFactory;
+use \CirrusSearch\SearchConfig;
use \Hooks;
/**
@@ -56,12 +58,19 @@
*/
private $optimizeForExperimentalHighlighter;
+ private $similarity;
+
/**
* Constructor
* @param bool $optimizeForExperimentalHighlighter should the index be
optimized for the experimental highlighter?
+ * @param SearchConfig $config
*/
- public function __construct( $optimizeForExperimentalHighlighter ) {
+ public function __construct( $optimizeForExperimentalHighlighter,
SearchConfig $config = null ) {
$this->optimizeForExperimentalHighlighter =
$optimizeForExperimentalHighlighter;
+ if ( is_null ( $config ) ) {
+ $config =
ConfigFactory::getDefaultInstance()->makeConfig( 'CirrusSearch' );
+ }
+ $this->similarity = $config->get(
'CirrusSearchSimilarityProfile' );
}
/**
@@ -79,11 +88,11 @@
// and is infered anyway.
$titleExtraAnalyzers = array(
$suggestExtra,
- array( 'index_analyzer' => 'prefix', 'search_analyzer'
=> 'near_match', 'index_options' => 'docs' ),
- array( 'index_analyzer' => 'prefix_asciifolding',
'search_analyzer' => 'near_match_asciifolding', 'index_options' => 'docs' ),
- array( 'analyzer' => 'near_match', 'index_options' =>
'docs' ),
- array( 'analyzer' => 'near_match_asciifolding',
'index_options' => 'docs' ),
- array( 'analyzer' => 'keyword', 'index_options' =>
'docs' ),
+ array( 'index_analyzer' => 'prefix', 'search_analyzer'
=> 'near_match', 'index_options' => 'docs', 'norms' => array( 'enabled' =>
false ) ),
+ array( 'index_analyzer' => 'prefix_asciifolding',
'search_analyzer' => 'near_match_asciifolding', 'index_options' => 'docs',
'norms' => array( 'enabled' => false ) ),
+ array( 'analyzer' => 'near_match', 'index_options' =>
'docs', 'norms' => array( 'enabled' => false ) ),
+ array( 'analyzer' => 'near_match_asciifolding',
'index_options' => 'docs', 'norms' => array( 'enabled' => false ) ),
+ array( 'analyzer' => 'keyword', 'index_options' =>
'docs', 'norms' => array( 'enabled' => false ) ),
);
if ( $flags & self::PREFIX_START_WITH_ANY ) {
$titleExtraAnalyzers[] = array(
@@ -118,24 +127,24 @@
),
'namespace' => $this->buildLongField(),
'namespace_text' => $this->buildKeywordField(),
- 'title' => $this->buildStringField(
+ 'title' => $this->buildStringField( 'title',
MappingConfigBuilder::ENABLE_NORMS |
MappingConfigBuilder::COPY_TO_SUGGEST,
$titleExtraAnalyzers ),
'text' => array_merge_recursive(
- $this->buildStringField( $textOptions,
$textExtraAnalyzers ),
+ $this->buildStringField( 'text',
$textOptions, $textExtraAnalyzers ),
array( 'fields' => array( 'word_count'
=> array(
'type' => 'token_count',
'store' => true,
'analyzer' => 'plain',
) ) )
),
- 'opening_text' => $this->buildStringField(
MappingConfigBuilder::ENABLE_NORMS ),
- 'auxiliary_text' => $this->buildStringField(
$textOptions ),
- 'file_text' => $this->buildStringField(
$textOptions ),
- 'source_text' => $this->buildStringField(
MappingConfigBuilder::MINIMAL,
+ 'opening_text' => $this->buildStringField(
'opening_text', MappingConfigBuilder::ENABLE_NORMS ),
+ 'auxiliary_text' => $this->buildStringField(
'auxiliary_text', $textOptions ),
+ 'file_text' => $this->buildStringField(
'file_text', $textOptions ),
+ 'source_text' => $this->buildStringField(
'source_text', MappingConfigBuilder::MINIMAL,
$sourceExtraAnalyzers
),
- 'category' => $this->buildStringField(
MappingConfigBuilder::SPEED_UP_HIGHLIGHTING, array(
+ 'category' => $this->buildStringField(
'category', $textOptions, array(
array(
'analyzer' =>
'lowercase_keyword',
'norms' => array( 'enabled' =>
false ),
@@ -146,14 +155,14 @@
'template' =>
$this->buildLowercaseKeywordField(),
'outgoing_link' => $this->buildKeywordField(),
'external_link' => $this->buildKeywordField(),
- 'heading' => $this->buildStringField(
MappingConfigBuilder::SPEED_UP_HIGHLIGHTING ),
+ 'heading' => $this->buildStringField(
'heading', MappingConfigBuilder::SPEED_UP_HIGHLIGHTING ),
'text_bytes' => $this->buildLongField( false ),
'redirect' => array(
'dynamic' => false,
'properties' => array(
'namespace' =>
$this->buildLongField(),
- 'title' =>
$this->buildStringField(
-
MappingConfigBuilder::COPY_TO_SUGGEST |
MappingConfigBuilder::SPEED_UP_HIGHLIGHTING,
+ 'title' =>
$this->buildStringField( 'redirect.title',
+ $textOptions |
MappingConfigBuilder::COPY_TO_SUGGEST,
$titleExtraAnalyzers ),
)
),
@@ -177,7 +186,7 @@
// This field can't be used for the fvh/experimental
highlighter for several reasons:
// 1. It is built with copy_to and not stored.
// 2. The term frequency information is all whoppy
compared to the "real" source text.
- $page[ 'properties' ][ 'all' ] =
$this->buildStringField( MappingConfigBuilder::ENABLE_NORMS );
+ $page[ 'properties' ][ 'all' ] =
$this->buildStringField( 'all', MappingConfigBuilder::ENABLE_NORMS );
$page = $this->setupCopyTo( $page,
$wgCirrusSearchWeights, 'all' );
// Now repeat for near_match fields. The same
considerations above apply except near_match
@@ -188,6 +197,7 @@
'index_options' => 'freqs',
'position_offset_gap' =>
self::POSITION_OFFSET_GAP,
'norms' => array( 'enabled' => false ),
+ 'similarity' => $this->getSimilarity(
'all_near_match' ),
'fields' => array(
'asciifolding' => array(
'type' => 'string',
@@ -195,6 +205,7 @@
'index_options' => 'freqs',
'position_offset_gap' =>
self::POSITION_OFFSET_GAP,
'norms' => array( 'enabled' =>
false ),
+ 'similarity' =>
$this->getSimilarity( 'all_near_match', 'asciifolding' ),
),
),
);
@@ -224,6 +235,28 @@
return $config;
}
+
+ /**
+ * Get the field similarity
+ * @param string $field
+ * @param string $analyzer
+ */
+ private function getSimilarity( $field, $analyzer = null ) {
+ $fieldSimilaraty = 'default';
+ if ( isset( $this->similarity['fields'] ) ) {
+ if( isset( $this->similarity['fields'][$field] ) ) {
+ $fieldSimilaraty =
$this->similarity['fields'][$field];
+ } else if ( $this->similarity['fields']['__default__']
) {
+ $fieldSimilaraty =
$this->similarity['fields']['__default__'];
+ }
+
+ if ( $analyzer != null && isset(
$this->similarity['fields']["$field.$analyzer"] ) ) {
+ $fieldSimilaraty =
$this->similarity['fields']["$field.$analyzer"];
+ }
+ }
+ return $fieldSimilaraty;
+ }
+
/**
* Setup copy_to for some fields to $destination.
* @param array $config to modify
@@ -250,6 +283,7 @@
/**
* Build a string field that does standard analysis for the language.
+ * @param string $fieldName the field name
* @param int $options Field options:
* ENABLE_NORMS: Enable norms on the field. Good for text you search
against but bad for array fields and useless
* for fields that don't get involved in the score.
@@ -259,19 +293,21 @@
* @param array $extra Extra analyzers for this field beyond the basic
text and plain.
* @return array definition of the field
*/
- public function buildStringField( $options, $extra = array() ) {
+ public function buildStringField( $fieldName, $options, $extra =
array() ) {
// multi_field is dead in 1.0 so we do this which actually
looks less gnarly.
$field = array(
'type' => 'string',
'index_analyzer' => 'text',
'search_analyzer' => 'text_search',
'position_offset_gap' => self::POSITION_OFFSET_GAP,
+ 'similarity' => $this->getSimilarity( $fieldName ),
'fields' => array(
'plain' => array(
'type' => 'string',
'index_analyzer' => 'plain',
'search_analyzer' => 'plain_search',
'position_offset_gap' =>
self::POSITION_OFFSET_GAP,
+ 'similarity' => $this->getSimilarity(
$fieldName, 'plain' ),
),
)
);
@@ -291,6 +327,7 @@
$extraName = $extraField[ 'index_analyzer' ];
}
$field[ 'fields' ][ $extraName ] = array_merge( array(
+ 'similarity' => $this->getSimilarity(
$fieldName, $extraName ),
'type' => 'string',
'position_offset_gap' =>
self::POSITION_OFFSET_GAP,
), $extraField );
diff --git a/profiles/SimilarityProfiles.php b/profiles/SimilarityProfiles.php
new file mode 100644
index 0000000..411dec8
--- /dev/null
+++ b/profiles/SimilarityProfiles.php
@@ -0,0 +1,81 @@
+<?php
+
+namespace CirrusSearch;
+/**
+ * CirrusSearch - List of Similarity profiles
+ * Configure the Similarity function used for scoring
+ * see
https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules-similarity.html
+ *
+ * A reindex is required when changes are made to the similarity configuration.
+ * NOTE the parameters that do not affect indexed values can be tuned manually
+ * in the index settings but it is possible only on a closed index.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ */
+
+$wgCirrusSearchSimilarityProfiles = array(
+ // default profile, uses the classic TF/IDF from Lucene
+ 'default' => array(),
+ // BM25 with default values for k and a for all fields
+ 'bm25_with_defaults' => array(
+ 'similarity' => array(
+ 'bm25' => array(
+ 'type' => 'BM25'
+ )
+ ),
+ 'fields' => array(
+ '__default__' => 'bm25',
+ )
+ ),
+ // Example with "randomly" tuned values
+ // (do not use)
+ 'bm25_tuned' => array(
+ 'similarity' => array(
+ 'title' => array(
+ 'type' => 'BM25',
+ 'k1' => 1.23,
+ 'b' => 0.75,
+ ),
+ 'opening' => array(
+ 'type' => 'BM25',
+ 'k1' => 1.22,
+ 'b' => 0.75,
+ ),
+ 'arrays' => array(
+ 'type' => 'BM25',
+ 'k1' => 1.1,
+ 'b' => 0.3,
+ ),
+ 'text' => array(
+ 'type' => 'BM25',
+ 'k1' => 1.3,
+ 'b' => 0.80,
+ ),
+ ),
+ 'fields' => array(
+ '__default__' => 'text',
+ // Field specific config
+ 'opening_text' => 'opening',
+ 'category' => 'arrays',
+ 'title' => 'title',
+ 'heading' => 'arrays',
+ // You can specify a custom config
+ // for a specific analyzer
+ 'title.plain' => 'short_fields',
+ 'redirect.title' => 'arrays',
+ ),
+ ),
+);
--
To view, visit https://gerrit.wikimedia.org/r/274982
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ie718c8a03ea5351e2e142f22436e2fd83314056f
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/CirrusSearch
Gerrit-Branch: master
Gerrit-Owner: DCausse <[email protected]>
Gerrit-Reviewer: Cindy-the-browser-test-bot <[email protected]>
Gerrit-Reviewer: EBernhardson <[email protected]>
Gerrit-Reviewer: Manybubbles <[email protected]>
Gerrit-Reviewer: Smalyshev <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
