This is an automated email from the ASF dual-hosted git repository.
snagel pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new dd94a61 NUTCH-2380 Upgrade indexer-elastic to Elasticsearch version
5.3.0 (contributed by Jurian Broertjes)
dd94a61 is described below
commit dd94a61d3359ede0e35480b26926901f25c4b250
Author: Sebastian Nagel <[email protected]>
AuthorDate: Sun Dec 17 12:18:37 2017 +0100
NUTCH-2380 Upgrade indexer-elastic to Elasticsearch version 5.3.0
(contributed by Jurian Broertjes)
---
src/plugin/indexer-elastic/ivy.xml | 6 +-
src/plugin/indexer-elastic/plugin.xml | 82 ++++++++++++++--------
.../indexwriter/elastic/ElasticIndexWriter.java | 11 +--
.../elastic/TestElasticIndexWriter.java | 13 ++--
4 files changed, 71 insertions(+), 41 deletions(-)
diff --git a/src/plugin/indexer-elastic/ivy.xml
b/src/plugin/indexer-elastic/ivy.xml
index f34075f..48ea594 100644
--- a/src/plugin/indexer-elastic/ivy.xml
+++ b/src/plugin/indexer-elastic/ivy.xml
@@ -36,8 +36,10 @@
</publications>
<dependencies>
- <dependency org="org.elasticsearch" name="elasticsearch" rev="2.3.3"
- conf="*->default"/>
+ <dependency org="org.elasticsearch" name="elasticsearch" rev="5.3.0"
conf="*->default"/>
+ <dependency org="org.elasticsearch.client" name="transport" rev="5.3.0"/>
+ <dependency org="org.apache.logging.log4j" name="log4j-api" rev="2.7" />
+ <dependency org="org.apache.logging.log4j" name="log4j-core" rev="2.7" />
</dependencies>
</ivy-module>
diff --git a/src/plugin/indexer-elastic/plugin.xml
b/src/plugin/indexer-elastic/plugin.xml
index 401e342..2cbb53a 100644
--- a/src/plugin/indexer-elastic/plugin.xml
+++ b/src/plugin/indexer-elastic/plugin.xml
@@ -22,38 +22,64 @@
<library name="indexer-elastic.jar">
<export name="*" />
</library>
- <library name="elasticsearch-2.3.3.jar"/>
- <library name="commons-cli-1.3.1.jar"/>
- <library name="compress-lzf-1.0.2.jar"/>
- <library name="guava-18.0.jar"/>
+
+ <!-- DEPENDENCIES FOR:
https://mvnrepository.com/artifact/org.elasticsearch.client/rest/5.3.0 -->
+ <library name="rest-5.3.0.jar"/>
+ <library name="commons-codec-1.10.jar"/>
+ <library name="commons-logging-1.1.3.jar"/>
+ <library name="httpasyncclient-4.1.2.jar"/>
+ <library name="httpclient-4.5.2.jar"/>
+ <library name="httpcore-4.4.5.jar"/>
+ <library name="httpcore-nio-4.4.5.jar"/>
+ <!-- ElasticSearch DEPENDENCIES-->
+ <library name="elasticsearch-5.3.0.jar"/>
+ <library name="compiler-0.9.3.jar"/>
<library name="HdrHistogram-2.1.6.jar"/>
<library name="hppc-0.7.1.jar"/>
- <library name="jackson-core-2.6.6.jar"/>
- <library name="jackson-dataformat-cbor-2.6.6.jar"/>
- <library name="jackson-dataformat-smile-2.6.6.jar"/>
- <library name="jackson-dataformat-yaml-2.6.6.jar"/>
- <library name="joda-convert-1.2.jar"/>
- <library name="joda-time-2.8.2.jar"/>
- <library name="jsr166e-1.1.0.jar"/>
- <library name="lucene-analyzers-common-5.5.0.jar"/>
- <library name="lucene-backward-codecs-5.5.0.jar"/>
- <library name="lucene-core-5.5.0.jar"/>
- <library name="lucene-grouping-5.5.0.jar"/>
- <library name="lucene-highlighter-5.5.0.jar"/>
- <library name="lucene-join-5.5.0.jar"/>
- <library name="lucene-memory-5.5.0.jar"/>
- <library name="lucene-misc-5.5.0.jar"/>
- <library name="lucene-queries-5.5.0.jar"/>
- <library name="lucene-queryparser-5.5.0.jar"/>
- <library name="lucene-sandbox-5.5.0.jar"/>
- <library name="lucene-spatial-5.5.0.jar"/>
- <library name="lucene-spatial3d-5.5.0.jar"/>
- <library name="lucene-suggest-5.5.0.jar"/>
- <library name="netty-3.10.5.Final.jar"/>
- <library name="securesm-1.0.jar"/>
+ <library name="jackson-core-2.8.6.jar"/>
+ <library name="jackson-dataformat-cbor-2.8.6.jar"/>
+ <library name="jackson-dataformat-smile-2.8.6.jar"/>
+ <library name="jackson-dataformat-yaml-2.8.6.jar"/>
+ <library name="jna-4.2.2.jar"/>
+ <library name="joda-time-2.9.5.jar"/>
+ <library name="jopt-simple-5.0.2.jar"/>
+ <library name="log4j-api-2.7.jar"/>
+ <library name="log4j-core-2.7.jar"/>
+ <library name="lucene-analyzers-common-6.4.1.jar"/>
+ <library name="lucene-backward-codecs-6.4.1.jar"/>
+ <library name="lucene-core-6.4.1.jar"/>
+ <library name="lucene-grouping-6.4.1.jar"/>
+ <library name="lucene-highlighter-6.4.1.jar"/>
+ <library name="lucene-join-6.4.1.jar"/>
+ <library name="lucene-memory-6.4.1.jar"/>
+ <library name="lucene-misc-6.4.1.jar"/>
+ <library name="lucene-queries-6.4.1.jar"/>
+ <library name="lucene-queryparser-6.4.1.jar"/>
+ <library name="lucene-sandbox-6.4.1.jar"/>
+ <library name="lucene-spatial3d-6.4.1.jar"/>
+ <library name="lucene-spatial-6.4.1.jar"/>
+ <library name="lucene-spatial-extras-6.4.1.jar"/>
+ <library name="lucene-suggest-6.4.1.jar"/>
+ <library name="securesm-1.1.jar"/>
<library name="snakeyaml-1.15.jar"/>
- <library name="spatial4j-0.5.jar"/>
<library name="t-digest-3.0.jar"/>
+
+ <!-- ElasticSearch Transport jar Deps-->
+ <library name="percolator-client-5.3.0.jar"/>
+ <library name="reindex-client-5.3.0.jar"/>
+ <library name="lang-mustache-client-5.3.0.jar"/>
+ <library name="transport-5.3.0.jar"/>
+ <library name="transport-netty3-client-5.3.0.jar"/>
+ <library name="transport-netty4-client-5.3.0.jar"/>
+ <library name="netty-all-4.1.7.Final.jar"/>
+ <library name="netty-3.10.6.Final.jar"/>
+ <library name="netty-buffer-4.1.7.Final.jar"/>
+ <library name="netty-codec-4.1.7.Final.jar"/>
+ <library name="netty-codec-http-4.1.7.Final.jar"/>
+ <library name="netty-common-4.1.7.Final.jar"/>
+ <library name="netty-handler-4.1.7.Final.jar"/>
+ <library name="netty-resolver-4.1.7.Final.jar"/>
+ <library name="netty-transport-4.1.7.Final.jar"/>
</runtime>
<requires>
diff --git
a/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java
b/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java
index 0132c91..d9671c4 100644
---
a/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java
+++
b/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java
@@ -17,8 +17,6 @@
package org.apache.nutch.indexwriter.elastic;
-import static org.elasticsearch.node.NodeBuilder.nodeBuilder;
-
import java.lang.invoke.MethodHandles;
import java.io.BufferedReader;
import java.io.IOException;
@@ -48,6 +46,8 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.node.Node;
+import org.elasticsearch.transport.client.*;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -108,7 +108,7 @@ public class ElasticIndexWriter implements IndexWriter {
String[] hosts = conf.getStrings(ElasticConstants.HOSTS);
int port = conf.getInt(ElasticConstants.PORT, DEFAULT_PORT);
- Settings.Builder settingsBuilder = Settings.settingsBuilder();
+ Settings.Builder settingsBuilder = Settings.builder();
BufferedReader reader = new BufferedReader(
conf.getConfResourceAsReader("elasticsearch.conf"));
@@ -135,12 +135,13 @@ public class ElasticIndexWriter implements IndexWriter {
// Prefer TransportClient
if (hosts != null && port > 1) {
- TransportClient transportClient =
TransportClient.builder().settings(settings).build();
+ TransportClient transportClient = new PreBuiltTransportClient(settings);
+
for (String host: hosts)
transportClient.addTransportAddress(new
InetSocketTransportAddress(InetAddress.getByName(host), port));
client = transportClient;
} else if (clusterName != null) {
- node = nodeBuilder().settings(settings).client(true).node();
+ node = new Node(settings);
client = node.client();
}
diff --git
a/src/plugin/indexer-elastic/src/test/org/apache/nutch/indexwriter/elastic/TestElasticIndexWriter.java
b/src/plugin/indexer-elastic/src/test/org/apache/nutch/indexwriter/elastic/TestElasticIndexWriter.java
index bae9737..8fab6a5 100644
---
a/src/plugin/indexer-elastic/src/test/org/apache/nutch/indexwriter/elastic/TestElasticIndexWriter.java
+++
b/src/plugin/indexer-elastic/src/test/org/apache/nutch/indexwriter/elastic/TestElasticIndexWriter.java
@@ -28,6 +28,7 @@ import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.action.DocWriteRequest.OpType;
import org.elasticsearch.action.bulk.BulkProcessor;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkItemResponse;
@@ -35,13 +36,14 @@ import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.support.AbstractClient;
-import org.elasticsearch.client.support.Headers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
+import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.threadpool.ThreadPool;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.util.UUID;
public class TestElasticIndexWriter {
@@ -63,10 +65,9 @@ public class TestElasticIndexWriter {
Settings settings = Settings.builder().build();
ThreadPool threadPool = new ThreadPool(settings);
- Headers headers = new Headers(settings);
// customize the ES client to simulate responses from an ES cluster
- client = new AbstractClient(settings, threadPool, headers) {
+ client = new AbstractClient(settings, threadPool) {
@Override
public void close() { }
@@ -84,14 +85,14 @@ public class TestElasticIndexWriter {
}
// respond with a failure
- BulkItemResponse failed = new BulkItemResponse(0, "index",
+ BulkItemResponse failed = new BulkItemResponse(0, OpType.INDEX,
new BulkItemResponse.Failure("nutch", "index", "failure0",
new EsRejectedExecutionException("saturated")));
response = new BulkResponse(new BulkItemResponse[]{failed}, 0);
} else {
// respond successfully
- BulkItemResponse success = new BulkItemResponse(0, "index",
- new IndexResponse("nutch", "index", "index0", 0, true));
+ BulkItemResponse success = new BulkItemResponse(0, OpType.INDEX,
+ new IndexResponse(new ShardId("nutch",
UUID.randomUUID().toString(), 0), "index", "index0", 0, true));
response = new BulkResponse(new BulkItemResponse[]{success}, 0);
}
--
To stop receiving notification emails like this one, please contact
['"[email protected]" <[email protected]>'].
