DCausse has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/337604 )
Change subject: Add support for elastic5
......................................................................
Add support for elastic5
Bug: T158070
Change-Id: I68a8a838bb1e53fd9ab26e2754f03a5104770441
---
M ttmserver/ElasticSearchTTMServer.php
M ttmserver/FuzzyLikeThis.php
2 files changed, 28 insertions(+), 52 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/mediawiki/extensions/Translate
refs/changes/04/337604/1
diff --git a/ttmserver/ElasticSearchTTMServer.php
b/ttmserver/ElasticSearchTTMServer.php
index b673539..d810ca5 100644
--- a/ttmserver/ElasticSearchTTMServer.php
+++ b/ttmserver/ElasticSearchTTMServer.php
@@ -200,6 +200,9 @@
$query->setSize( $sizeSecond );
// Break if we already got all hits
+ // FIXME: we should set a hard limit that can be lower
+ // than getTotalHits(), we may encounter the 10000
+ // offset+size hard limit by elasticsearch.
} while ( $resultset->getTotalHits() > count( $contents ) );
$suggestions = array();
@@ -382,27 +385,27 @@
$mapping = new \Elastica\Type\Mapping();
$mapping->setType( $type );
$mapping->setProperties( array(
- 'wiki' => array( 'type' => 'string', 'index' =>
'not_analyzed' ),
- 'localid' => array( 'type' => 'string', 'index' =>
'not_analyzed' ),
- 'uri' => array( 'type' => 'string', 'index' =>
'not_analyzed' ),
- 'language' => array( 'type' => 'string', 'index' =>
'not_analyzed' ),
- 'group' => array( 'type' => 'string', 'index' =>
'not_analyzed' ),
+ 'wiki' => array( 'type' => 'keyword' ),
+ 'localid' => array( 'type' => 'keyword' ),
+ 'uri' => array( 'type' => 'keyword' ),
+ 'language' => array( 'type' => 'keyword' ),
+ 'group' => array( 'type' => 'keyword' ),
'content' => array(
- 'type' => 'string',
+ 'type' => 'text',
'fields' => array(
'content' => array(
- 'type' => 'string',
+ 'type' => 'text',
'index' => 'analyzed',
'term_vector' => 'yes'
),
'prefix_complete' => array(
- 'type' => 'string',
+ 'type' => 'text',
'analyzer' => 'prefix',
'search_analyzer' => 'standard',
'term_vector' => 'yes'
),
'case_sensitive' => array(
- 'type' => 'string',
+ 'type' => 'text',
'index' => 'analyzed',
'analyzer' => 'casesensitive',
'term_vector' => 'yes'
@@ -457,7 +460,7 @@
public function endBootstrap() {
$index = $this->getType()->getIndex();
$index->refresh();
- $index->optimize();
+ $index->forcemerge();
$index->getSettings()->setRefreshInterval( '5s' );
}
@@ -752,26 +755,23 @@
*/
private function deleteByQuery( \Elastica\Type $type, \Elastica\Query
$query ) {
$retryAttempts = self::BULK_INDEX_RETRY_ATTEMPTS;
- $scrollOptions = array(
- 'search_type' => 'scan',
- 'scroll' => '15m',
- 'size' => self::BULK_DELETE_CHUNK_SIZE,
- );
- $result = $type->search( $query, $scrollOptions );
- MWElasticUtils::iterateOverScroll( $type->getIndex(),
- $result->getResponse()->getScrollId(), '15m',
- function( $results ) use( $retryAttempts, $type ) {
- $ids = array();
- foreach ( $results as $result ) {
- $ids[] = $result->getId();
+ $search = new \Elastica\Search( $this->getClient() );
+ $search->setQuery( $query );
+ $search->addType( $type );
+ $scroll = new \Elastica\Scroll( $search, '15m' );
+
+ foreach( $scroll as $results ) {
+ $ids = array();
+ foreach ( $results as $result ) {
+ $ids[] = $result->getId();
+ }
+ MWElasticUtils::withRetry( $retryAttempts,
+ function() use ( $ids, $type ) {
+ $type->deleteIds( $ids );
}
- MWElasticUtils::withRetry( $retryAttempts,
- function() use ( $ids, $type ) {
- $type->deleteIds( $ids );
- }
- );
- }, 0, $retryAttempts );
+ );
+ }
}
/**
diff --git a/ttmserver/FuzzyLikeThis.php b/ttmserver/FuzzyLikeThis.php
index 592ac34..96fe331 100644
--- a/ttmserver/FuzzyLikeThis.php
+++ b/ttmserver/FuzzyLikeThis.php
@@ -90,13 +90,6 @@
protected $_prefixLength = 0;
/**
- * Boost.
- *
- * @var float Boost
- */
- protected $_boost = 1.0;
-
- /**
* Analyzer.
*
* @var sting Analyzer
@@ -159,19 +152,6 @@
}
/**
- * Set boost.
- *
- * @param float $value Boost value
- *
- * @return $this
- */
- public function setBoost( $value ) {
- $this->_boost = (float) $value;
-
- return $this;
- }
-
- /**
* Set Prefix Length.
*
* @param int $value Prefix length
@@ -221,10 +201,6 @@
public function toArray() {
if ( !empty( $this->_fields ) ) {
$args['fields'] = $this->_fields;
- }
-
- if ( !empty( $this->_boost ) ) {
- $args['boost'] = $this->_boost;
}
if ( !empty( $this->_analyzer ) ) {
--
To view, visit https://gerrit.wikimedia.org/r/337604
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I68a8a838bb1e53fd9ab26e2754f03a5104770441
Gerrit-PatchSet: 1
Gerrit-Project: mediawiki/extensions/Translate
Gerrit-Branch: master
Gerrit-Owner: DCausse <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
