Tjones has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/343089 )
Change subject: Enable ICU Folding for Swedish
......................................................................
Enable ICU Folding for Swedish
Unpack Swedish built-in analyzer and enable ICU folding.
Change language code for Swedish folding exceptions in
ICUFoldingUnicodeSetFilter from sw to sv.
Disable default dev ICUFoldingUnicodeSetFilter ICU folding filter
test (which happened to be for Russian) so that the expected
language-specific exceptions can apply.
Bug: T160562
Change-Id: I555d1ea268d3ea141b94b5834274d753496c89fd
---
M includes/Maintenance/AnalysisConfigBuilder.php
M tests/jenkins/FullyFeaturedConfig.php
2 files changed, 28 insertions(+), 5 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/CirrusSearch
refs/changes/89/343089/1
diff --git a/includes/Maintenance/AnalysisConfigBuilder.php
b/includes/Maintenance/AnalysisConfigBuilder.php
index a42f87e..1a73e47 100644
--- a/includes/Maintenance/AnalysisConfigBuilder.php
+++ b/includes/Maintenance/AnalysisConfigBuilder.php
@@ -315,7 +315,7 @@
// @todo: complete the default filters per language
case 'fi': return '[^åäöÅÄÖ]';
case 'ru': return '[^йЙ]';
- case 'sw': return '[^åäöÅÄÖ]';
+ case 'sv': return '[^åäöÅÄÖ]';
default: return null;
}
}
@@ -800,6 +800,33 @@
// In Russian text_search is just a copy of text
$config[ 'analyzer' ][ 'text_search' ] = $config[
'analyzer' ][ 'text' ];
break;
+ case 'swedish':
+ // Add asciifolding_preserve to filters
+ $config[ 'analyzer' ][ 'lowercase_keyword' ][ 'filter'
][] = 'asciifolding_preserve';
+
+ // Unpack built-in swedish analyzer to add
asciifolding_preserve
+ $config['filter']['swedish_stop'] = [
+ 'type' => 'stop',
+ 'stopwords' => '_swedish_',
+ ];
+ $config['filter']['swedish_stemmer'] = [
+ 'type' => 'stemmer',
+ 'language' => 'swedish',
+ ];
+
+ $config['analyzer']['text'] = [
+ 'tokenizer' => 'standard',
+ 'filter' => [
+ 'lowercase',
+ 'swedish_stop',
+ 'swedish_stemmer',
+ 'asciifolding_preserve',
+ ],
+ ];
+
+ // In Swedish text_search is just a copy of text
+ $config['analyzer']['text_search'] =
$config['analyzer']['text'];
+ break;
case 'turkish':
$config[ 'filter' ][ 'lowercase' ][ 'language' ] =
'turkish';
break;
diff --git a/tests/jenkins/FullyFeaturedConfig.php
b/tests/jenkins/FullyFeaturedConfig.php
index 57104ed..94d6414 100644
--- a/tests/jenkins/FullyFeaturedConfig.php
+++ b/tests/jenkins/FullyFeaturedConfig.php
@@ -33,10 +33,6 @@
$wgCirrusSearchWikimediaExtraPlugin[ 'id_hash_mod_filter' ] = true;
$wgCirrusSearchWikimediaExtraPlugin[ 'documentVersion' ] = true;
$wgCirrusSearchUseIcuFolding = 'yes';
-// Test ICU folding filter
-// see full_text_api.feature: Searching without accents finds results with
accents
-$wgCirrusSearchICUFoldingUnicodeSetFilter = '[^йЙ]';
-
$wgCirrusSearchUseCompletionSuggester = 'yes';
$wgCirrusSearchCompletionSuggesterUseDefaultSort = true;
--
To view, visit https://gerrit.wikimedia.org/r/343089
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I555d1ea268d3ea141b94b5834274d753496c89fd
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/CirrusSearch
Gerrit-Branch: master
Gerrit-Owner: Tjones <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
