jenkins-bot has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/343089 )
Change subject: Enable ICU Folding for Swedish
......................................................................
Enable ICU Folding for Swedish
Unpack Swedish built-in analyzer and enable ICU folding.
Change language code for Swedish folding exceptions in
ICUFoldingUnicodeSetFilter from sw to sv.
Update AnalysisConfigBuilderTest results for sv.
Update FullyFeaturedConfig:
- Disable default dev ICUFoldingUnicodeSetFilter ICU folding filter
test (which happened to be for Russian) so that the expected
language-specific exceptions can apply.
- Remove $wgCirrusSearchUseIcuFolding so that it is not on by default
in dev since it is not on by default in prod.
Move й/и from default test wiki to Russian test wiki
Bug: T160562
Change-Id: I555d1ea268d3ea141b94b5834274d753496c89fd
---
M includes/Maintenance/AnalysisConfigBuilder.php
M tests/browser/features/full_text_api.feature
M tests/browser/features/russian_folding.feature
M tests/jenkins/FullyFeaturedConfig.php
M tests/unit/fixtures/languageAnalysis/sv.expected
5 files changed, 65 insertions(+), 15 deletions(-)
Approvals:
Smalyshev: Looks good to me, but someone else must approve
Cindy-the-browser-test-bot: Looks good to me, but someone else must approve
jenkins-bot: Verified
DCausse: Looks good to me, approved
diff --git a/includes/Maintenance/AnalysisConfigBuilder.php
b/includes/Maintenance/AnalysisConfigBuilder.php
index a42f87e..1735928 100644
--- a/includes/Maintenance/AnalysisConfigBuilder.php
+++ b/includes/Maintenance/AnalysisConfigBuilder.php
@@ -315,7 +315,7 @@
// @todo: complete the default filters per language
case 'fi': return '[^åäöÅÄÖ]';
case 'ru': return '[^йЙ]';
- case 'sw': return '[^åäöÅÄÖ]';
+ case 'sv': return '[^åäöÅÄÖ]';
default: return null;
}
}
@@ -800,6 +800,34 @@
// In Russian text_search is just a copy of text
$config[ 'analyzer' ][ 'text_search' ] = $config[
'analyzer' ][ 'text' ];
break;
+ case 'swedish':
+ // Add asciifolding_preserve to filters
+ $config[ 'analyzer' ][ 'lowercase_keyword' ][ 'filter'
][] = 'asciifolding_preserve';
+
+ // Unpack built-in swedish analyzer to add
asciifolding_preserve
+ $config['filter']['swedish_stop'] = [
+ 'type' => 'stop',
+ 'stopwords' => '_swedish_',
+ ];
+ $config['filter']['swedish_stemmer'] = [
+ 'type' => 'stemmer',
+ 'language' => 'swedish',
+ ];
+
+ $config['analyzer']['text'] = [
+ 'type' => 'custom',
+ 'tokenizer' => 'standard',
+ 'filter' => [
+ 'lowercase',
+ 'swedish_stop',
+ 'swedish_stemmer',
+ 'asciifolding_preserve',
+ ],
+ ];
+
+ // In Swedish text_search is just a copy of text
+ $config['analyzer']['text_search'] =
$config['analyzer']['text'];
+ break;
case 'turkish':
$config[ 'filter' ][ 'lowercase' ][ 'language' ] =
'turkish';
break;
@@ -947,6 +975,7 @@
'simple' => true,
'fr' => true,
'he' => true,
+ 'sv' => true,
];
/**
diff --git a/tests/browser/features/full_text_api.feature
b/tests/browser/features/full_text_api.feature
index a31d5c1..9ebcab5 100644
--- a/tests/browser/features/full_text_api.feature
+++ b/tests/browser/features/full_text_api.feature
@@ -133,8 +133,6 @@
| term | result |
| ανθρωπος | Page with non ascii letters is the first api search result
|
| ἄνθρωπος | Page with non ascii letters is the first api search result
|
- | широкии | there are no api search results
|
- | широкий | Page with non ascii letters is the first api search result
|
@accented_namespace
Scenario: Searching for an accented namespace without the accent finds
things in it
diff --git a/tests/browser/features/russian_folding.feature
b/tests/browser/features/russian_folding.feature
index e8185b4..6cc6dae 100644
--- a/tests/browser/features/russian_folding.feature
+++ b/tests/browser/features/russian_folding.feature
@@ -21,3 +21,11 @@
Scenario: Searching for with accent
When I api search on ru for Бра́зер
Then Бразер is the first api search result
+
+ Scenario: Searching for й when text has й
+ When I api search on ru for чёрный
+ Then Саша Чёрный is the first api search result
+
+ Scenario: Searching for й when text has и
+ When I api search on ru for чёрныи
+ Then there are no api search results
diff --git a/tests/jenkins/FullyFeaturedConfig.php
b/tests/jenkins/FullyFeaturedConfig.php
index 57104ed..c035d52 100644
--- a/tests/jenkins/FullyFeaturedConfig.php
+++ b/tests/jenkins/FullyFeaturedConfig.php
@@ -32,11 +32,6 @@
$wgCirrusSearchWikimediaExtraPlugin[ 'super_detect_noop' ] = true;
$wgCirrusSearchWikimediaExtraPlugin[ 'id_hash_mod_filter' ] = true;
$wgCirrusSearchWikimediaExtraPlugin[ 'documentVersion' ] = true;
-$wgCirrusSearchUseIcuFolding = 'yes';
-// Test ICU folding filter
-// see full_text_api.feature: Searching without accents finds results with
accents
-$wgCirrusSearchICUFoldingUnicodeSetFilter = '[^йЙ]';
-
$wgCirrusSearchUseCompletionSuggester = 'yes';
$wgCirrusSearchCompletionSuggesterUseDefaultSort = true;
diff --git a/tests/unit/fixtures/languageAnalysis/sv.expected
b/tests/unit/fixtures/languageAnalysis/sv.expected
index 2b5029a..df7cab3 100644
--- a/tests/unit/fixtures/languageAnalysis/sv.expected
+++ b/tests/unit/fixtures/languageAnalysis/sv.expected
@@ -1,15 +1,25 @@
{
"analyzer": {
"text": {
- "type": "swedish",
- "char_filter": [
- "word_break_helper"
+ "type": "custom",
+ "tokenizer": "standard",
+ "filter": [
+ "lowercase",
+ "swedish_stop",
+ "swedish_stemmer",
+ "asciifolding_preserve",
+ "dedup_asciifolding"
]
},
"text_search": {
- "type": "swedish",
- "char_filter": [
- "word_break_helper"
+ "type": "custom",
+ "tokenizer": "standard",
+ "filter": [
+ "lowercase",
+ "swedish_stop",
+ "swedish_stemmer",
+ "asciifolding_preserve",
+ "dedup_asciifolding"
]
},
"plain": {
@@ -166,7 +176,9 @@
"tokenizer": "no_splitting",
"filter": [
"truncate_keyword",
- "lowercase"
+ "lowercase",
+ "asciifolding_preserve",
+ "dedup_asciifolding"
]
},
"trigram": {
@@ -212,6 +224,14 @@
"type": "truncate",
"length": 5000
},
+ "swedish_stop": {
+ "type": "stop",
+ "stopwords": "_swedish_"
+ },
+ "swedish_stemmer": {
+ "type": "stemmer",
+ "language": "swedish"
+ },
"dedup_asciifolding": {
"type": "unique",
"only_on_same_position": true
--
To view, visit https://gerrit.wikimedia.org/r/343089
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I555d1ea268d3ea141b94b5834274d753496c89fd
Gerrit-PatchSet: 4
Gerrit-Project: mediawiki/extensions/CirrusSearch
Gerrit-Branch: master
Gerrit-Owner: Tjones <[email protected]>
Gerrit-Reviewer: Cindy-the-browser-test-bot <[email protected]>
Gerrit-Reviewer: DCausse <[email protected]>
Gerrit-Reviewer: EBernhardson <[email protected]>
Gerrit-Reviewer: Gehel <[email protected]>
Gerrit-Reviewer: Smalyshev <[email protected]>
Gerrit-Reviewer: Tjones <[email protected]>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
