DCausse has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/375963 )
Change subject: Upgrade to elastic 5.5
......................................................................
Upgrade to elastic 5.5
NativeScripts are deprecated: added a new ScriptService to handle the super
noop script.
The old native script is kept while cirrus transitions to the new script.
Bug: T174652
Change-Id: I172b18983b631b04325dde74519aa228e0a6ef4d
---
M pom.xml
M src/main/java/org/wikimedia/search/extra/ExtraPlugin.java
M
src/main/java/org/wikimedia/search/extra/latency/TransportLatencyStatsAction.java
M
src/main/java/org/wikimedia/search/extra/superdetectnoop/SuperDetectNoopScript.java
A src/test/java/org/wikimedia/search/extra/MockPluginWithoutNativeScript.java
M src/test/java/org/wikimedia/search/extra/regex/SourceRegexBuilderESTest.java
M
src/test/java/org/wikimedia/search/extra/router/DegradedRouterBuilderESTest.java
M
src/test/java/org/wikimedia/search/extra/router/TokenCountRouterBuilderESTest.java
M
src/test/java/org/wikimedia/search/extra/superdetectnoop/SuperDetectNoopScriptIntegrationTest.java
9 files changed, 144 insertions(+), 56 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/search/extra
refs/changes/63/375963/1
diff --git a/pom.xml b/pom.xml
index d726401..0bef134 100644
--- a/pom.xml
+++ b/pom.xml
@@ -10,7 +10,7 @@
<groupId>org.wikimedia.search</groupId>
<artifactId>extra</artifactId>
- <version>5.4.0-SNAPSHOT</version>
+ <version>5.5.2-SNAPSHOT</version>
<description>Extra queries and filters for Elasticsearch.</description>
<licenses>
@@ -44,8 +44,8 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <elasticsearch.version>5.4.0</elasticsearch.version>
- <lucene.version>6.5.0</lucene.version>
+ <elasticsearch.version>5.5.2</elasticsearch.version>
+ <lucene.version>6.6.0</lucene.version>
<randomizedtesting.version>2.5.0</randomizedtesting.version>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
@@ -380,7 +380,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.11</version>
+ <version>4.12</version>
<scope>test</scope>
<exclusions>
<exclusion>
diff --git a/src/main/java/org/wikimedia/search/extra/ExtraPlugin.java
b/src/main/java/org/wikimedia/search/extra/ExtraPlugin.java
index 5d33dc4..147064c 100644
--- a/src/main/java/org/wikimedia/search/extra/ExtraPlugin.java
+++ b/src/main/java/org/wikimedia/search/extra/ExtraPlugin.java
@@ -23,6 +23,7 @@
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestHandler;
import org.elasticsearch.script.NativeScriptFactory;
+import org.elasticsearch.script.ScriptEngineService;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.watcher.ResourceWatcherService;
@@ -45,15 +46,18 @@
import org.wikimedia.search.extra.superdetectnoop.WithinPercentageHandler;
import org.wikimedia.search.extra.util.Suppliers.MutableSupplier;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.function.Supplier;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.unmodifiableList;
+import static java.util.Collections.unmodifiableSet;
/**
@@ -64,11 +68,20 @@
private final SearchLatencyListener latencyListener;
private final MutableSupplier<ThreadPool> threadPoolSupplier;
private final LoadService loadStats;
+ private final SuperDetectNoopScript.SuperNoopScriptEngineService
superDetectNoopService;
public ExtraPlugin(Settings settings) {
threadPoolSupplier = new MutableSupplier<>();
latencyListener = new SearchLatencyListener(settings,
threadPoolSupplier);
loadStats = new LoadService(latencyListener, new OsService(settings));
+ superDetectNoopService = new
SuperDetectNoopScript.SuperNoopScriptEngineService(
+ unmodifiableSet(new HashSet<>(asList(
+ new ChangeHandler.Equal.Recognizer(),
+ new WithinPercentageHandler.Recognizer(),
+ new WithinAbsoluteHandler.Recognizer(),
+ new SetHandler.Recognizer(),
+ new VersionedDocumentHandler.Recognizer())
+ )));
}
@Override
@@ -76,7 +89,7 @@
ResourceWatcherService
resourceWatcherService, ScriptService scriptService,
NamedXContentRegistry
xContentRegistry) {
threadPoolSupplier.set(threadPool);
- return Collections.singletonList(latencyListener);
+ return singletonList(latencyListener);
}
/**
@@ -85,7 +98,7 @@
@Override
@SuppressWarnings("deprecation")
public List<QuerySpec<?>> getQueries() {
- return Arrays.asList(
+ return asList(
new QuerySpec<>(SourceRegexQueryBuilder.NAME,
SourceRegexQueryBuilder::new, SourceRegexQueryBuilder::fromXContent),
new QuerySpec<>(FuzzyLikeThisQueryBuilder.NAME,
FuzzyLikeThisQueryBuilder::new, FuzzyLikeThisQueryBuilder::fromXContent),
new QuerySpec<>(TokenCountRouterQueryBuilder.NAME,
TokenCountRouterQueryBuilder::new, TokenCountRouterQueryBuilder::fromXContent),
@@ -102,20 +115,20 @@
}
@Override
+ @Deprecated
+ @SuppressWarnings("deprecation")
public List<NativeScriptFactory> getNativeScripts() {
- Set<ChangeHandler.Recognizer> recognizers = new
HashSet<>(Arrays.asList(
- new ChangeHandler.Equal.Recognizer(),
- new WithinPercentageHandler.Recognizer(),
- new WithinAbsoluteHandler.Recognizer(),
- new SetHandler.Recognizer(),
- new VersionedDocumentHandler.Recognizer()
- ));
- return Collections.singletonList(new
SuperDetectNoopScript.Factory(recognizers));
+ return singletonList(new
SuperDetectNoopScript.SuperNoopNativeScriptFactory(superDetectNoopService));
+ }
+
+ @Override
+ public ScriptEngineService getScriptEngineService(Settings settings) {
+ return superDetectNoopService;
}
@Override
public List<ScoreFunctionSpec<?>> getScoreFunctions() {
- return Collections.singletonList(
+ return singletonList(
new ScoreFunctionSpec<>(
LevenshteinDistanceScoreBuilder.NAME.getPreferredName(),
LevenshteinDistanceScoreBuilder::new,
@@ -131,7 +144,7 @@
@Override
public List<ActionHandler<? extends ActionRequest, ? extends
ActionResponse>> getActions() {
- return Collections.unmodifiableList(Collections.singletonList(
+ return unmodifiableList(singletonList(
new ActionHandler<>(LatencyStatsAction.INSTANCE,
TransportLatencyStatsAction.class)
));
}
@@ -141,6 +154,6 @@
ClusterSettings clusterSettings,
IndexScopedSettings indexScopedSettings,
SettingsFilter settingsFilter,
IndexNameExpressionResolver indexNameExpressionResolver,
Supplier<DiscoveryNodes>
nodesInCluster) {
- return Collections.singletonList(new RestGetLatencyStats(settings,
restController));
+ return singletonList(new RestGetLatencyStats(settings,
restController));
}
}
diff --git
a/src/main/java/org/wikimedia/search/extra/latency/TransportLatencyStatsAction.java
b/src/main/java/org/wikimedia/search/extra/latency/TransportLatencyStatsAction.java
index b9dd045..57107e5 100644
---
a/src/main/java/org/wikimedia/search/extra/latency/TransportLatencyStatsAction.java
+++
b/src/main/java/org/wikimedia/search/extra/latency/TransportLatencyStatsAction.java
@@ -54,11 +54,6 @@
return new
LatencyStatsNodeResponse(clusterService.localNode()).initFromProbe(latencyProbe);
}
- @Override
- protected boolean accumulateExceptions() {
- return false;
- }
-
static class LatencyStatsNodeRequest extends BaseNodeRequest {
LatencyStatsNodeRequest() {
diff --git
a/src/main/java/org/wikimedia/search/extra/superdetectnoop/SuperDetectNoopScript.java
b/src/main/java/org/wikimedia/search/extra/superdetectnoop/SuperDetectNoopScript.java
index a8a7153..3d182dd 100644
---
a/src/main/java/org/wikimedia/search/extra/superdetectnoop/SuperDetectNoopScript.java
+++
b/src/main/java/org/wikimedia/search/extra/superdetectnoop/SuperDetectNoopScript.java
@@ -1,9 +1,14 @@
package org.wikimedia.search.extra.superdetectnoop;
import org.elasticsearch.script.AbstractExecutableScript;
+import org.elasticsearch.script.CompiledScript;
import org.elasticsearch.script.ExecutableScript;
import org.elasticsearch.script.NativeScriptFactory;
+import org.elasticsearch.script.ScriptEngineService;
+import org.elasticsearch.script.SearchScript;
+import org.elasticsearch.search.lookup.SearchLookup;
+import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
@@ -17,22 +22,68 @@
* detectors! So much power!
*/
public class SuperDetectNoopScript extends AbstractExecutableScript {
- public static class Factory implements NativeScriptFactory {
- private final Set<ChangeHandler.Recognizer> changeHandlerRecognizers;
- public Factory(Set<ChangeHandler.Recognizer> recognizers) {
- // Note that recognizers are tried in a random order....
- this.changeHandlerRecognizers =
Collections.unmodifiableSet(recognizers);
+ @Deprecated
+ @SuppressWarnings("deprecation")
+ public static class SuperNoopNativeScriptFactory implements
NativeScriptFactory {
+ private final SuperNoopScriptEngineService service;
+ public SuperNoopNativeScriptFactory(SuperNoopScriptEngineService
service) {
+ this.service = service;
}
@Override
- public ExecutableScript newScript(Map<String, Object> params) {
- @SuppressWarnings("unchecked")
- Map<String, Object> source = (Map<String, Object>)
params.get("source");
- return new SuperDetectNoopScript(source, handlers(params));
+ public ExecutableScript newScript(Map<String, Object> map) {
+ return service.newScript(map);
}
- private Map<String, ChangeHandler<Object>> handlers(Map<String,
Object> params) {
+ @Override
+ public boolean needsScores() {
+ return false;
+ }
+
+ @Override
+ public String getName() {
+ return "super_detect_noop";
+ }
+ }
+
+ public static class SuperNoopScriptEngineService implements
ScriptEngineService {
+ private final Set<ChangeHandler.Recognizer> changeHandlerRecognizers;
+
+ public SuperNoopScriptEngineService(Set<ChangeHandler.Recognizer>
changeHandlerRecognizers) {
+ this.changeHandlerRecognizers = changeHandlerRecognizers;
+ }
+
+ @Override
+ public String getType() {
+ return "super_detect_noop";
+ }
+
+ @Override
+ public Object compile(String scriptName, String scriptSource,
Map<String, String> map) {
+ return "super_detect_noop (compiled script is useless)";
+ }
+
+ @Override
+ public ExecutableScript executable(CompiledScript compiledScript,
Map<String, Object> map) {
+ return newScript(map);
+ }
+
+ @Override
+ public SearchScript search(CompiledScript compiledScript, SearchLookup
searchLookup, Map<String, Object> map) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean isInlineScriptEnabled() {
+ return true;
+ }
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ protected Map<String, ChangeHandler<Object>> handlers(Map<String,
Object> params) {
@SuppressWarnings("unchecked")
Map<String, String> detectorConfigs = (Map<String, String>)
params.get("handlers");
if (detectorConfigs == null) {
@@ -45,7 +96,7 @@
return Collections.unmodifiableMap(handlers);
}
- private ChangeHandler<Object> handler(String config) {
+ protected ChangeHandler<Object> handler(String config) {
for (ChangeHandler.Recognizer factory : changeHandlerRecognizers) {
ChangeHandler<Object> detector = factory.build(config);
if (detector != null) {
@@ -55,14 +106,10 @@
throw new IllegalArgumentException("Don't recognize this type of
change handler: " + config);
}
- @Override
- public boolean needsScores() {
- return false;
- }
-
- @Override
- public String getName() {
- return "super_detect_noop";
+ public ExecutableScript newScript(Map<String, Object> params) {
+ @SuppressWarnings("unchecked")
+ Map<String, Object> source = (Map<String, Object>)
params.get("source");
+ return new SuperDetectNoopScript(source, handlers(params));
}
}
diff --git
a/src/test/java/org/wikimedia/search/extra/MockPluginWithoutNativeScript.java
b/src/test/java/org/wikimedia/search/extra/MockPluginWithoutNativeScript.java
new file mode 100644
index 0000000..e5ee7f4
--- /dev/null
+++
b/src/test/java/org/wikimedia/search/extra/MockPluginWithoutNativeScript.java
@@ -0,0 +1,24 @@
+package org.wikimedia.search.extra;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.script.NativeScriptFactory;
+
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Needed for some AbstractQueryBuilderTest
+ * native script will generate a warning at startup and will cause
+ * failure on ESTestCase internal assertions.
+ */
+public class MockPluginWithoutNativeScript extends ExtraPlugin {
+ public MockPluginWithoutNativeScript(Settings settings) {
+ super(settings);
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public List<NativeScriptFactory> getNativeScripts() {
+ return Collections.emptyList();
+ }
+}
diff --git
a/src/test/java/org/wikimedia/search/extra/regex/SourceRegexBuilderESTest.java
b/src/test/java/org/wikimedia/search/extra/regex/SourceRegexBuilderESTest.java
index 2fa7b3c..e02cc88 100644
---
a/src/test/java/org/wikimedia/search/extra/regex/SourceRegexBuilderESTest.java
+++
b/src/test/java/org/wikimedia/search/extra/regex/SourceRegexBuilderESTest.java
@@ -12,7 +12,7 @@
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.test.AbstractQueryTestCase;
-import org.wikimedia.search.extra.ExtraPlugin;
+import org.wikimedia.search.extra.MockPluginWithoutNativeScript;
import org.wikimedia.search.extra.router.TokenCountRouterQueryBuilder;
import org.wikimedia.search.extra.util.FieldValues;
@@ -27,7 +27,7 @@
public class SourceRegexBuilderESTest extends
AbstractQueryTestCase<SourceRegexQueryBuilder> {
protected Collection<Class<? extends Plugin>> getPlugins() {
- return Collections.singleton(ExtraPlugin.class);
+ return Collections.singleton(MockPluginWithoutNativeScript.class);
}
private static final String MY_FIELD = "regex_field";
private static final String MY_FIELD_NGRAM = "regex_field_ngram";
diff --git
a/src/test/java/org/wikimedia/search/extra/router/DegradedRouterBuilderESTest.java
b/src/test/java/org/wikimedia/search/extra/router/DegradedRouterBuilderESTest.java
index 7a0a2d3..c781eae 100644
---
a/src/test/java/org/wikimedia/search/extra/router/DegradedRouterBuilderESTest.java
+++
b/src/test/java/org/wikimedia/search/extra/router/DegradedRouterBuilderESTest.java
@@ -3,17 +3,20 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.index.memory.MemoryIndex;
import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
-import org.elasticsearch.index.query.*;
+import org.elasticsearch.index.query.MatchNoneQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.index.query.WrapperQueryBuilder;
import org.elasticsearch.monitor.os.OsService;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.test.AbstractQueryTestCase;
import org.junit.runner.RunWith;
-import org.wikimedia.search.extra.ExtraPlugin;
+import org.wikimedia.search.extra.MockPluginWithoutNativeScript;
import org.wikimedia.search.extra.latency.SearchLatencyProbe;
import
org.wikimedia.search.extra.router.AbstractRouterQueryBuilder.ConditionDefinition;
import
org.wikimedia.search.extra.router.DegradedRouterQueryBuilder.DegradedConditionType;
@@ -25,13 +28,13 @@
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.instanceOf;
-import static
org.wikimedia.search.extra.router.AbstractRouterQueryBuilder.ConditionDefinition.gt;
import static org.mockito.Mockito.mock;
+import static
org.wikimedia.search.extra.router.AbstractRouterQueryBuilder.ConditionDefinition.gt;
@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class)
public class DegradedRouterBuilderESTest extends
AbstractQueryTestCase<DegradedRouterQueryBuilder>{
protected Collection<Class<? extends Plugin>> getPlugins() {
- return Collections.singleton(ExtraPlugin.class);
+ return Collections.singleton(MockPluginWithoutNativeScript.class);
}
@Override
diff --git
a/src/test/java/org/wikimedia/search/extra/router/TokenCountRouterBuilderESTest.java
b/src/test/java/org/wikimedia/search/extra/router/TokenCountRouterBuilderESTest.java
index b568b42..aafa826 100644
---
a/src/test/java/org/wikimedia/search/extra/router/TokenCountRouterBuilderESTest.java
+++
b/src/test/java/org/wikimedia/search/extra/router/TokenCountRouterBuilderESTest.java
@@ -4,11 +4,11 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.index.memory.MemoryIndex;
import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.compress.CompressedXContent;
-import org.elasticsearch.common.lucene.search.MatchNoDocsQuery;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.MatchNoneQueryBuilder;
@@ -21,7 +21,7 @@
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.test.AbstractQueryTestCase;
-import org.wikimedia.search.extra.ExtraPlugin;
+import org.wikimedia.search.extra.MockPluginWithoutNativeScript;
import org.wikimedia.search.extra.router.AbstractRouterQueryBuilder.Condition;
import java.io.IOException;
@@ -37,7 +37,7 @@
public class TokenCountRouterBuilderESTest extends
AbstractQueryTestCase<TokenCountRouterQueryBuilder> {
protected Collection<Class<? extends Plugin>> getPlugins() {
- return Collections.singleton(ExtraPlugin.class);
+ return Collections.singleton(MockPluginWithoutNativeScript.class);
}
private static final String MY_FIELD = "tok_count_field";
diff --git
a/src/test/java/org/wikimedia/search/extra/superdetectnoop/SuperDetectNoopScriptIntegrationTest.java
b/src/test/java/org/wikimedia/search/extra/superdetectnoop/SuperDetectNoopScriptIntegrationTest.java
index b156b9e..aa7d75f 100644
---
a/src/test/java/org/wikimedia/search/extra/superdetectnoop/SuperDetectNoopScriptIntegrationTest.java
+++
b/src/test/java/org/wikimedia/search/extra/superdetectnoop/SuperDetectNoopScriptIntegrationTest.java
@@ -10,6 +10,7 @@
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptType;
@@ -531,9 +532,14 @@
private UpdateRequestBuilder toUpdateRequest(XContentBuilder b) {
b.close();
- Map<String, Object> m = XContentHelper.convertToMap(b.bytes(),
true).v2();
- return client().prepareUpdate("test", "test", "1").setScript(new
Script(ScriptType.INLINE, "native", "super_detect_noop", m))
+ Map<String, Object> m = XContentHelper.convertToMap(b.bytes(), true,
XContentType.JSON).v2();
+ Script script;
+ if (random().nextInt(3) > 0) {
+ script = new Script(ScriptType.INLINE, "super_detect_noop", "", m);
+ } else {
+ script = new Script(ScriptType.INLINE, "native",
"super_detect_noop", m);
+ }
+ return client().prepareUpdate("test", "test", "1").setScript(script)
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
-
}
}
--
To view, visit https://gerrit.wikimedia.org/r/375963
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I172b18983b631b04325dde74519aa228e0a6ef4d
Gerrit-PatchSet: 1
Gerrit-Project: search/extra
Gerrit-Branch: master
Gerrit-Owner: DCausse <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
