[
https://issues.apache.org/jira/browse/HBASE-27782?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17736185#comment-17736185
]
Duo Zhang commented on HBASE-27782:
-----------------------------------
OK, it is very easy to reproduce...
{code}
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.ipc;
import java.io.IOException;
import java.net.ServerSocket;
import java.net.Socket;
import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.MetricsConnection.CallStats;
import org.apache.hadoop.hbase.io.crypto.tls.X509Util;
import org.apache.hadoop.hbase.net.Address;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.io.Closeables;
@Category({ ClientTests.class, SmallTests.class })
public class TestTLSHandshadeFailure {
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestTLSHandshadeFailure.class);
private static final Logger LOG =
LoggerFactory.getLogger(TestTLSHandshadeFailure.class);
// use a pre set seed to make the random bytes stable
private final Random rand = new Random(1);
private ServerSocket server;
private Thread serverThread;
private NettyRpcClient client;
private void serve() {
Socket socket = null;
try {
socket = server.accept();
byte[] bytes = new byte[128];
rand.nextBytes(bytes);
socket.getOutputStream().write(bytes);
socket.getOutputStream().flush();
} catch (Exception e) {
LOG.warn("failed to process request", e);
} finally {
if (socket != null) {
try {
socket.close();
} catch (IOException e1) {
LOG.warn("failed to close socket");
}
}
}
}
@Before
public void setUp() throws IOException {
server = new ServerSocket(0);
serverThread = new Thread(this::serve);
serverThread.setDaemon(true);
serverThread.setName("Error-Server-Thread");
serverThread.start();
Configuration conf = HBaseConfiguration.create();
conf.setBoolean(X509Util.HBASE_CLIENT_NETTY_TLS_ENABLED, true);
client = new NettyRpcClient(conf);
}
@After
public void tearDown() throws IOException {
Closeables.close(client, true);
Closeables.close(server, true);
}
@Test
public void test() throws IOException {
ConnectionId id = new ConnectionId(User.getCurrent(), "test",
Address.fromParts("127.0.0.1", server.getLocalPort()));
NettyRpcConnection conn = client.createConnection(id);
BlockingRpcCallback<Call> done = new BlockingRpcCallback<>();
Call call = new Call(1, null, null, null, null, 0, 0, done, new
CallStats());
HBaseRpcController hrc = new HBaseRpcControllerImpl();
conn.sendRequest(call, hrc);
done.get();
call.error.printStackTrace();
}
}
{code}
I can see exactly the same error output
{noformat}
2023-06-22T23:09:46,176 WARN [RPCClient-NioEventLoopGroup-1-1 {}]
channel.DefaultChannelPipeline(1152): An exceptionCaught() event was fired, and
it reached at the tail of the pipeline. It usually means the last handler in
the pipeline did not handle the exception.
org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException:
org.apache.hbase.thirdparty.io.netty.handler.ssl.NotSslRecordException: not an
SSL/TLS record:
73d51abbd89cb8196f0efb6892f94d68fccc2c35f0b84609e5f12c55dd85aba8d5d9bef76808f3b572e5900112b81927ba5bb5f67e1bda28b4049bf0e4aed78db15d7bf2fc0c34e9a99de4ef3bc2b17c8137ad659878f9e93df1f658367aca286452474b9ef3765e24e9a88173724dddfb04b01dcceb0c8aead641c58dad5695
at
org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:499)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
~[hbase-shaded-netty-4.1.4.jar:?]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_372]
Caused by:
org.apache.hbase.thirdparty.io.netty.handler.ssl.NotSslRecordException: not an
SSL/TLS record:
73d51abbd89cb8196f0efb6892f94d68fccc2c35f0b84609e5f12c55dd85aba8d5d9bef76808f3b572e5900112b81927ba5bb5f67e1bda28b4049bf0e4aed78db15d7bf2fc0c34e9a99de4ef3bc2b17c8137ad659878f9e93df1f658367aca286452474b9ef3765e24e9a88173724dddfb04b01dcceb0c8aead641c58dad5695
at
org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1215)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1285)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
~[hbase-shaded-netty-4.1.4.jar:?]
at
org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
~[hbase-shaded-netty-4.1.4.jar:?]
... 17 more
{noformat}
Let me think how to make the error message not so confusing.
> During SSL handshake error, netty complains that exceptionCaught() was not
> handled
> ----------------------------------------------------------------------------------
>
> Key: HBASE-27782
> URL: https://issues.apache.org/jira/browse/HBASE-27782
> Project: HBase
> Issue Type: Bug
> Components: Client, rpc, security
> Reporter: Bryan Beaudreault
> Assignee: Duo Zhang
> Priority: Major
> Labels: TLS
>
> I was chaos testing the new native TLS, forcing a certificate to expire and
> fail handshake. The handshake failure properly causes submitted requests to
> fail, but I see the following "unhandled exception" like message:
> {code:java}
> WARN o.a.h.t.i.n.c.DefaultChannelPipeline - An exceptionCaught() event was
> fired, and it reached at the tail of the pipeline. It usually means the last
> handler in the pipeline did not handle the exception.
> org.apache.hbase.thirdparty.io.netty.handler.codec.DecoderException:
> javax.net.ssl.SSLHandshakeException: Received fatal alert: certificate_expired
> at
> org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:499)
> at
> org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
> at
> org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
> at
> org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
> at
> org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
> at
> org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410)
> at
> org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440)
> at
> org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
> at
> org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919)
> at
> org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166)
> at
> org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788)
> at
> org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724)
> at
> org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650)
> at
> org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562)
> at
> org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
> at
> org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
> at
> org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
> at java.base/java.lang.Thread.run(Thread.java:833)
> Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert:
> certificate_expired
> at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:131)
> at java.base/sun.security.ssl.Alert.createSSLException(Alert.java:117)
> at
> java.base/sun.security.ssl.TransportContext.fatal(TransportContext.java:358)
> at
> java.base/sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:293)
> at
> java.base/sun.security.ssl.TransportContext.dispatch(TransportContext.java:204)
> at
> java.base/sun.security.ssl.SSLTransport.decode(SSLTransport.java:172)
> at
> java.base/sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:736)
> at
> java.base/sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:691)
> at
> java.base/sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:506)
> at
> java.base/sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:482)
> at java.base/javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:679)
> at
> org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:296)
> at
> org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1343)
> at
> org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1236)
> at
> org.apache.hbase.thirdparty.io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1285)
> at
> org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:529)
> at
> org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:468)
> ... 17 common frames omitted{code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
