This is an automated email from the ASF dual-hosted git repository. snagel pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push: new 01ab00b6c NUTCH-2953 Indexer Elastic to ignore SSL issues - apply patch contributed by Markus Jelsma - fix class imports 01ab00b6c is described below commit 01ab00b6cd8dbba8abbf1d3840a09bab929c6af0 Author: Sebastian Nagel <sna...@apache.org> AuthorDate: Mon Aug 8 16:19:24 2022 +0200 NUTCH-2953 Indexer Elastic to ignore SSL issues - apply patch contributed by Markus Jelsma - fix class imports --- .../indexwriter/elastic/ElasticIndexWriter.java | 31 +++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java b/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java index 7885a5210..053bfd68a 100644 --- a/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java +++ b/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/ElasticIndexWriter.java @@ -25,14 +25,20 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import javax.net.ssl.SSLContext; + import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.TrustSelfSignedStrategy; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.ssl.SSLContexts; import org.apache.nutch.indexer.IndexWriter; import org.apache.nutch.indexer.IndexWriterParams; import org.apache.nutch.indexer.NutchDocument; @@ -181,6 +187,7 @@ public class ElasticIndexWriter implements IndexWriter { hostsList[i++] = new HttpHost(host, port, scheme); } RestClientBuilder restClientBuilder = RestClient.builder(hostsList); + if (auth) { restClientBuilder .setHttpClientConfigCallback(new HttpClientConfigCallback() { @@ -191,6 +198,28 @@ public class ElasticIndexWriter implements IndexWriter { } }); } + + // In case of HTTPS, set the client up for ignoring problems with self-signed + // certificates and stuff + if ("https".equals(scheme)) { + try { + SSLContextBuilder sslBuilder = SSLContexts.custom(); + sslBuilder.loadTrustMaterial(null, new TrustSelfSignedStrategy()); + final SSLContext sslContext = sslBuilder.build(); + + restClientBuilder.setHttpClientConfigCallback(new HttpClientConfigCallback() { + @Override + public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { + // ignore issues with self-signed certificates + httpClientBuilder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE); + return httpClientBuilder.setSSLContext(sslContext); + } + }); + } catch (Exception e) { + LOG.error("Error setting up SSLContext because: " + e.getMessage(), e); + } + } + client = new RestHighLevelClient(restClientBuilder); } else { throw new IOException( @@ -344,4 +373,4 @@ public class ElasticIndexWriter implements IndexWriter { public Configuration getConf() { return config; } -} \ No newline at end of file +}