This is an automated email from the ASF dual-hosted git repository.
markus pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
The following commit(s) were added to refs/heads/master by this push:
new 61d7e8c NUTCH-2647 Skip TLS certificate checks in protocol-http plugin
61d7e8c is described below
commit 61d7e8ce440aa544ce23e98a6fc6f811c482c5a0
Author: Markus Jelsma <mar...@apache.org>
AuthorDate: Fri Sep 28 11:25:31 2018 +0200
NUTCH-2647 Skip TLS certificate checks in protocol-http plugin
---
.../nutch/protocol/http/DummyX509TrustManager.java | 93 ++++++++++++++++++++++
.../apache/nutch/protocol/http/HttpResponse.java | 14 ++--
2 files changed, 102 insertions(+), 5 deletions(-)
diff --git
a/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/DummyX509TrustManager.java
b/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/DummyX509TrustManager.java
new file mode 100644
index 0000000..879f703
--- /dev/null
+++
b/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/DummyX509TrustManager.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/*
+ * Based on EasyX509TrustManager from commons-httpclient.
+ */
+
+package org.apache.nutch.protocol.http;
+
+import java.lang.invoke.MethodHandles;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DummyX509TrustManager implements X509TrustManager {
+ private X509TrustManager standardTrustManager = null;
+
+ /** Logger object for this class. */
+ private static final Logger LOG = LoggerFactory
+ .getLogger(MethodHandles.lookup().lookupClass());
+
+ /**
+ * Constructor for DummyX509TrustManager.
+ */
+ public DummyX509TrustManager(KeyStore keystore)
+ throws NoSuchAlgorithmException, KeyStoreException {
+ super();
+ String algo = TrustManagerFactory.getDefaultAlgorithm();
+ TrustManagerFactory factory = TrustManagerFactory.getInstance(algo);
+ factory.init(keystore);
+ TrustManager[] trustmanagers = factory.getTrustManagers();
+ if (trustmanagers.length == 0) {
+ throw new NoSuchAlgorithmException(algo + " trust manager not
supported");
+ }
+ this.standardTrustManager = (X509TrustManager) trustmanagers[0];
+ }
+
+ /**
+ * @see javax.net.ssl.X509TrustManager#checkClientTrusted(X509Certificate[],
+ * String)
+ */
+ public boolean isClientTrusted(X509Certificate[] certificates) {
+ return true;
+ }
+
+ /**
+ * @see javax.net.ssl.X509TrustManager#checkServerTrusted(X509Certificate[],
+ * String)
+ */
+ public boolean isServerTrusted(X509Certificate[] certificates) {
+ return true;
+ }
+
+ /**
+ * @see javax.net.ssl.X509TrustManager#getAcceptedIssuers()
+ */
+ public X509Certificate[] getAcceptedIssuers() {
+ return this.standardTrustManager.getAcceptedIssuers();
+ }
+
+ public void checkClientTrusted(X509Certificate[] arg0, String arg1)
+ throws CertificateException {
+ // do nothing
+
+ }
+
+ public void checkServerTrusted(X509Certificate[] arg0, String arg1)
+ throws CertificateException {
+ // do nothing
+
+ }
+}
diff --git
a/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java
b/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java
index 4b5544e..95ae352 100644
---
a/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java
+++
b/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java
@@ -30,8 +30,10 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
+import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocket;
import javax.net.ssl.SSLSocketFactory;
+import javax.net.ssl.TrustManager;
import org.apache.hadoop.io.Text;
import org.apache.nutch.crawl.CrawlDatum;
@@ -131,7 +133,7 @@ public class HttpResponse implements Response {
try {
sslsocket = getSSLSocket(socket, sockHost, sockPort);
sslsocket.startHandshake();
- } catch (IOException e) {
+ } catch (Exception e) {
Http.LOG.debug("SSL connection to {} failed with: {}", url,
e.getMessage());
if ("handshake alert: unrecognized_name".equals(e.getMessage())) {
@@ -142,7 +144,7 @@ public class HttpResponse implements Response {
socket.connect(sockAddr, http.getTimeout());
sslsocket = getSSLSocket(socket, "", sockPort);
sslsocket.startHandshake();
- } catch (IOException ex) {
+ } catch (Exception ex) {
String msg = "SSL reconnect to " + url + " failed with: "
+ e.getMessage();
throw new HttpException(msg);
@@ -353,9 +355,11 @@ public class HttpResponse implements Response {
* -------------------------
*/
- private SSLSocket getSSLSocket(Socket socket, String sockHost, int sockPort)
throws IOException {
- SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory
- .getDefault();
+ private SSLSocket getSSLSocket(Socket socket, String sockHost, int sockPort)
throws Exception {
+ SSLContext sslContext = SSLContext.getInstance("TLS");
+ sslContext.init(null, new TrustManager[]{new DummyX509TrustManager(null)},
null);
+ SSLSocketFactory factory = sslContext.getSocketFactory();
+
SSLSocket sslsocket = (SSLSocket) factory
.createSocket(socket, sockHost, sockPort, true);
sslsocket.setUseClientMode(true);
