This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git
The following commit(s) were added to refs/heads/master by this push: new fa10b7d ISSUE #1103: Add channel TLS counters fa10b7d is described below commit fa10b7dcd89c40222ba5f30bb60f785bd21669b2 Author: Kishore Udayashankar <kudayashan...@salesforce.com> AuthorDate: Mon Feb 19 00:10:53 2018 -0800 ISSUE #1103: Add channel TLS counters (bug W-4482570) Add counters to track - active TLS - active non-TLS channel connections - failed TLS handshakes - failed connections Also, Added tests to verify counters Drain pendingOps on TLS handshake failure Master Issue: #1103 Author: Kishore Udayashankar <kudayashan...@salesforce.com> Author: kishorekasi <kkasi...@gmail.com> Reviewers: Sijie Guo <si...@apache.org>, Venkateswararao Jujjuri (JV) <None> This closes #1106 from kishorekasi/tls-stats, closes #1103 --- .../bookkeeper/client/BookKeeperClientStats.java | 5 + .../bookkeeper/proto/PerChannelBookieClient.java | 41 ++++- .../apache/bookkeeper/tls/TLSContextFactory.java | 15 +- .../java/org/apache/bookkeeper/tls/TestTLS.java | 203 +++++++++++++++++---- 4 files changed, 229 insertions(+), 35 deletions(-) diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperClientStats.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperClientStats.java index e3d8438..7d0fd5d 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperClientStats.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/BookKeeperClientStats.java @@ -69,9 +69,14 @@ public interface BookKeeperClientStats { String TIMEOUT_GET_BOOKIE_INFO = "TIMEOUT_GET_BOOKIE_INFO"; String CHANNEL_START_TLS_OP = "START_TLS"; String CHANNEL_TIMEOUT_START_TLS_OP = "TIMEOUT_START_TLS"; + String NETTY_EXCEPTION_CNT = "NETTY_EXCEPTION_CNT"; String CLIENT_CONNECT_TIMER = "CLIENT_CONNECT_TIMER"; String ADD_OP_OUTSTANDING = "ADD_OP_OUTSTANDING"; String READ_OP_OUTSTANDING = "READ_OP_OUTSTANDING"; String NETTY_OPS = "NETTY_OPS"; + String ACTIVE_NON_TLS_CHANNEL_COUNTER = "ACTIVE_NON_TLS_CHANNEL_COUNTER"; + String ACTIVE_TLS_CHANNEL_COUNTER = "ACTIVE_TLS_CHANNEL_COUNTER"; + String FAILED_CONNECTION_COUNTER = "FAILED_CONNECTION_COUNTER"; + String FAILED_TLS_HANDSHAKE_COUNTER = "FAILED_TLS_HANDSHAKE_COUNTER"; } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java index 666b644..1173d77 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/PerChannelBookieClient.java @@ -50,6 +50,7 @@ import io.netty.channel.epoll.EpollSocketChannel; import io.netty.channel.local.LocalChannel; import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.handler.codec.CorruptedFrameException; +import io.netty.handler.codec.DecoderException; import io.netty.handler.codec.LengthFieldBasedFrameDecoder; import io.netty.handler.codec.LengthFieldPrepender; import io.netty.handler.codec.TooLongFrameException; @@ -76,6 +77,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.BiPredicate; +import javax.net.ssl.SSLHandshakeException; import javax.net.ssl.SSLPeerUnverifiedException; import org.apache.bookkeeper.auth.BookKeeperPrincipal; @@ -181,6 +183,10 @@ public class PerChannelBookieClient extends ChannelInboundHandlerAdapter { private final Counter readEntryOutstanding; /* collect stats on all Ops that flows through netty pipeline */ private final OpStatsLogger nettyOpLogger; + private final Counter activeNonTlsChannelCounter; + private final Counter activeTlsChannelCounter; + private final Counter failedConnectionCounter; + private final Counter failedTlsHandshakeCounter; private final boolean useV2WireProtocol; @@ -283,6 +289,10 @@ public class PerChannelBookieClient extends ChannelInboundHandlerAdapter { addEntryOutstanding = statsLogger.getCounter(BookKeeperClientStats.ADD_OP_OUTSTANDING); readEntryOutstanding = statsLogger.getCounter(BookKeeperClientStats.READ_OP_OUTSTANDING); nettyOpLogger = statsLogger.getOpStatsLogger(BookKeeperClientStats.NETTY_OPS); + activeNonTlsChannelCounter = statsLogger.getCounter(BookKeeperClientStats.ACTIVE_NON_TLS_CHANNEL_COUNTER); + activeTlsChannelCounter = statsLogger.getCounter(BookKeeperClientStats.ACTIVE_TLS_CHANNEL_COUNTER); + failedConnectionCounter = statsLogger.getCounter(BookKeeperClientStats.FAILED_CONNECTION_COUNTER); + failedTlsHandshakeCounter = statsLogger.getCounter(BookKeeperClientStats.FAILED_TLS_HANDSHAKE_COUNTER); this.pcbcPool = pcbcPool; @@ -837,6 +847,13 @@ public class PerChannelBookieClient extends ChannelInboundHandlerAdapter { } finally { closeLock.writeLock().unlock(); } + + if (channel != null && channel.pipeline().get(SslHandler.class) != null) { + activeTlsChannelCounter.dec(); + } else { + activeNonTlsChannelCounter.dec(); + } + closeInternal(true, wait); } @@ -857,13 +874,13 @@ public class PerChannelBookieClient extends ChannelInboundHandlerAdapter { cf.awaitUninterruptibly(); } } - } private ChannelFuture closeChannel(Channel c) { if (LOG.isDebugEnabled()) { LOG.debug("Closing channel {}", c); } + return c.close(); } @@ -941,7 +958,7 @@ public class PerChannelBookieClient extends ChannelInboundHandlerAdapter { /** * Errors out pending ops from per channel bookie client. As the channel * is being closed, all the operations waiting on the connection - * will be sent to completion with error + * will be sent to completion with error. */ void errorOutPendingOps(int rc) { Queue<GenericCallback<PerChannelBookieClient>> oldPendingOps; @@ -993,6 +1010,11 @@ public class PerChannelBookieClient extends ChannelInboundHandlerAdapter { LOG.info("Disconnected from bookie channel {}", ctx.channel()); if (ctx.channel() != null) { closeChannel(ctx.channel()); + if (ctx.channel().pipeline().get(SslHandler.class) != null) { + activeTlsChannelCounter.dec(); + } else { + activeNonTlsChannelCounter.dec(); + } } errorOutOutstandingEntries(BKException.Code.BookieHandleNotAvailableException); @@ -1032,6 +1054,15 @@ public class PerChannelBookieClient extends ChannelInboundHandlerAdapter { return; } + if (cause instanceof DecoderException && cause.getCause() instanceof SSLHandshakeException) { + LOG.error("TLS handshake failed", cause); + errorOutPendingOps(BKException.Code.SecurityException); + Channel c = ctx.channel(); + if (c != null) { + closeChannel(c); + } + } + if (cause instanceof IOException) { LOG.warn("Exception caught on:{} cause:", ctx.channel(), cause); ctx.close(); @@ -1255,6 +1286,7 @@ public class PerChannelBookieClient extends ChannelInboundHandlerAdapter { AuthHandler.ClientSideHandler authHandler = future.get().pipeline() .get(AuthHandler.ClientSideHandler.class); authHandler.authProvider.onProtocolUpgrade(); + activeTlsChannelCounter.inc(); } else if (future.isSuccess() && (state == ConnectionState.CLOSED || state == ConnectionState.DISCONNECTED)) { LOG.warn("Closed before TLS handshake completed, clean up: {}, current state {}", @@ -1276,6 +1308,7 @@ public class PerChannelBookieClient extends ChannelInboundHandlerAdapter { if (state != ConnectionState.CLOSED) { state = ConnectionState.DISCONNECTED; } + failedTlsHandshakeCounter.inc(); } // trick to not do operations under the lock, take the list @@ -2011,6 +2044,7 @@ public class PerChannelBookieClient extends ChannelInboundHandlerAdapter { } else { LOG.info("Successfully connected to bookie: " + addr); state = ConnectionState.CONNECTED; + activeNonTlsChannelCounter.inc(); } } else if (future.isSuccess() && state == ConnectionState.START_TLS) { rc = BKException.Code.OK; @@ -2020,6 +2054,7 @@ public class PerChannelBookieClient extends ChannelInboundHandlerAdapter { AuthHandler.ClientSideHandler authHandler = future.channel().pipeline() .get(AuthHandler.ClientSideHandler.class); authHandler.authProvider.onProtocolUpgrade(); + activeTlsChannelCounter.inc(); } else if (future.isSuccess() && (state == ConnectionState.CLOSED || state == ConnectionState.DISCONNECTED)) { LOG.warn("Closed before connection completed, clean up: {}, current state {}", @@ -2041,6 +2076,7 @@ public class PerChannelBookieClient extends ChannelInboundHandlerAdapter { if (state != ConnectionState.CLOSED) { state = ConnectionState.DISCONNECTED; } + failedConnectionCounter.inc(); } // trick to not do operations under the lock, take the list @@ -2086,5 +2122,6 @@ public class PerChannelBookieClient extends ChannelInboundHandlerAdapter { for (GenericCallback<PerChannelBookieClient> pendingOp : oldPendingOps) { pendingOp.operationComplete(rc, null); } + failedTlsHandshakeCounter.inc(); } } diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tls/TLSContextFactory.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tls/TLSContextFactory.java index 2d3f51b..17aea85 100644 --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/tls/TLSContextFactory.java +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/tls/TLSContextFactory.java @@ -58,7 +58,20 @@ public class TLSContextFactory implements SecurityHandlerFactory { * Supported Key File Types. */ public enum KeyStoreType { - PKCS12, JKS, PEM; + PKCS12("PKCS12"), + JKS("JKS"), + PEM("PEM"); + + private String str; + + KeyStoreType(String str) { + this.str = str; + } + + @Override + public String toString() { + return this.str; + } } private static final Logger LOG = LoggerFactory.getLogger(TLSContextFactory.class); diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/tls/TestTLS.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/tls/TestTLS.java index 31ea7bd..d4a480f 100644 --- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/tls/TestTLS.java +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/tls/TestTLS.java @@ -17,6 +17,7 @@ */ package org.apache.bookkeeper.tls; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -24,6 +25,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; +import java.net.InetSocketAddress; import java.security.cert.Certificate; import java.security.cert.X509Certificate; import java.util.ArrayList; @@ -42,6 +44,8 @@ import org.apache.bookkeeper.client.BKException; import org.apache.bookkeeper.client.BookKeeper; import org.apache.bookkeeper.client.BookKeeper.DigestType; import org.apache.bookkeeper.client.BookKeeperAdmin; +import org.apache.bookkeeper.client.BookKeeperClientStats; +import org.apache.bookkeeper.client.BookKeeperTestClient; import org.apache.bookkeeper.client.LedgerEntry; import org.apache.bookkeeper.client.LedgerHandle; import org.apache.bookkeeper.client.LedgerMetadata; @@ -49,9 +53,11 @@ import org.apache.bookkeeper.conf.ClientConfiguration; import org.apache.bookkeeper.conf.ServerConfiguration; import org.apache.bookkeeper.net.BookieSocketAddress; import org.apache.bookkeeper.proto.BookieConnectionPeer; +import org.apache.bookkeeper.proto.BookieServer; import org.apache.bookkeeper.proto.ClientConnectionPeer; import org.apache.bookkeeper.proto.TestPerChannelBookieClient; import org.apache.bookkeeper.test.BookKeeperClusterTestCase; +import org.apache.bookkeeper.test.TestStatsProvider; import org.apache.bookkeeper.tls.TLSContextFactory.KeyStoreType; import org.junit.After; import org.junit.Assume; @@ -281,27 +287,31 @@ public class TestTLS extends BookKeeperClusterTestCase { } } - private LedgerMetadata testClient(ClientConfiguration conf, int clusterSize) throws Exception { - try (BookKeeper client = new BookKeeper(conf);) { - byte[] passwd = "testPassword".getBytes(); - int numEntries = 100; - long lid; - byte[] testEntry = "testEntry".getBytes(); - try (LedgerHandle lh = client.createLedger(clusterSize, clusterSize, DigestType.CRC32, passwd);) { - for (int i = 0; i <= numEntries; i++) { - lh.addEntry(testEntry); - } - lid = lh.getId(); + private LedgerMetadata testClient(BookKeeper client, int clusterSize) throws Exception { + byte[] passwd = "testPassword".getBytes(); + int numEntries = 100; + long lid; + byte[] testEntry = "testEntry".getBytes(); + try (LedgerHandle lh = client.createLedger(clusterSize, clusterSize, DigestType.CRC32, passwd);) { + for (int i = 0; i <= numEntries; i++) { + lh.addEntry(testEntry); } - try (LedgerHandle lh = client.openLedger(lid, DigestType.CRC32, passwd);) { - Enumeration<LedgerEntry> entries = lh.readEntries(0, numEntries); - while (entries.hasMoreElements()) { - LedgerEntry e = entries.nextElement(); - assertTrue("Entry contents incorrect", Arrays.equals(e.getEntry(), testEntry)); - } - BookKeeperAdmin admin = new BookKeeperAdmin(client); - return admin.getLedgerMetadata(lh); + lid = lh.getId(); + } + try (LedgerHandle lh = client.openLedger(lid, DigestType.CRC32, passwd);) { + Enumeration<LedgerEntry> entries = lh.readEntries(0, numEntries); + while (entries.hasMoreElements()) { + LedgerEntry e = entries.nextElement(); + assertTrue("Entry contents incorrect", Arrays.equals(e.getEntry(), testEntry)); } + BookKeeperAdmin admin = new BookKeeperAdmin(client); + return admin.getLedgerMetadata(lh); + } + } + + private LedgerMetadata testClient(ClientConfiguration conf, int clusterSize) throws Exception { + try (BookKeeper client = new BookKeeper(conf);) { + return testClient(client, clusterSize); } } @@ -567,7 +577,7 @@ public class TestTLS extends BookKeeperClusterTestCase { secureBookieSideChannel = false; secureBookieSideChannelPrincipals = null; ClientConfiguration clientConf = new ClientConfiguration(baseClientConf); - clientConf.setTLSProviderFactoryClass(""); + clientConf.setTLSProviderFactoryClass(null); try { testClient(clientConf, numBookies); @@ -671,18 +681,6 @@ public class TestTLS extends BookKeeperClusterTestCase { int origNumBookies = numBookies; ServerConfiguration bookieConf = newServerConfiguration(); - /* - bookieConf.setTLSProviderFactoryClass(null); - bs.add(startBookie(bookieConf)); - try { - testClient(clientConf, origNumBookies + 1); - fail("Shouldn't be able to connect"); - } catch (BKException.BKNotEnoughBookiesException nnbe) { - // correct response - } - - bookieConf = newServerConfiguration(); - */ bookieConf.setTLSProviderFactoryClass(TLSContextFactory.class.getName()); bs.add(startBookie(bookieConf)); testClient(clientConf, origNumBookies + 1); @@ -705,4 +703,145 @@ public class TestTLS extends BookKeeperClusterTestCase { LOG.info("latch countdown"); latch.countDown(); } + + /** + * Verify TLS and non-TLS channel counters. + */ + @Test + public void testTLSChannelCounters() throws Exception { + ClientConfiguration tlsClientconf = new ClientConfiguration(baseClientConf) + .setNumChannelsPerBookie(1); + ClientConfiguration nonTlsClientconf = new ClientConfiguration(baseClientConf) + .setNumChannelsPerBookie(1) + .setTLSProviderFactoryClass(null); + + TestStatsProvider tlsStatsProvider = new TestStatsProvider(); + TestStatsProvider nonTlsStatsProvider = new TestStatsProvider(); + BookKeeperTestClient tlsClient = new BookKeeperTestClient(tlsClientconf, tlsStatsProvider); + BookKeeperTestClient nonTlsClient = new BookKeeperTestClient(nonTlsClientconf, nonTlsStatsProvider); + + // IO load from clients + testClient(tlsClient, numBookies); + testClient(nonTlsClient, numBookies); + + // verify stats + for (int i = 0; i < numBookies; i++) { + BookieServer bookie = bs.get(i); + InetSocketAddress addr = bookie.getLocalAddress().getSocketAddress(); + StringBuilder nameBuilder = new StringBuilder(BookKeeperClientStats.CHANNEL_SCOPE) + .append(".") + .append(addr.getAddress().getHostAddress() + .replace('.', '_') + .replace('-', '_')) + .append("_") + .append(addr.getPort()) + .append("."); + + // check stats on TLS enabled client + assertEquals("Mismatch TLS channel count", 1, + tlsClient.getTestStatsProvider().getCounter(nameBuilder.toString() + + BookKeeperClientStats.ACTIVE_TLS_CHANNEL_COUNTER).get().longValue()); + assertEquals("TLS handshake failure unexpected", 0, + tlsClient.getTestStatsProvider().getCounter(nameBuilder.toString() + + BookKeeperClientStats.FAILED_TLS_HANDSHAKE_COUNTER).get().longValue()); + assertEquals("Mismatch non-TLS channel count", 0, + tlsClient.getTestStatsProvider().getCounter(nameBuilder.toString() + + BookKeeperClientStats.ACTIVE_NON_TLS_CHANNEL_COUNTER).get().longValue()); + assertEquals("Connection failures unexpected", 0, + tlsClient.getTestStatsProvider().getCounter(nameBuilder.toString() + + BookKeeperClientStats.FAILED_CONNECTION_COUNTER).get().longValue()); + + // check stats on non-TLS enabled client + assertEquals("Mismatch TLS channel count", 0, + nonTlsClient.getTestStatsProvider().getCounter(nameBuilder.toString() + + BookKeeperClientStats.ACTIVE_TLS_CHANNEL_COUNTER).get().longValue()); + assertEquals("TLS handshake failure unexpected", 0, + nonTlsClient.getTestStatsProvider().getCounter(nameBuilder.toString() + + BookKeeperClientStats.FAILED_TLS_HANDSHAKE_COUNTER).get().longValue()); + assertEquals("Mismatch non-TLS channel count", 1, + nonTlsClient.getTestStatsProvider().getCounter(nameBuilder.toString() + + BookKeeperClientStats.ACTIVE_NON_TLS_CHANNEL_COUNTER).get().longValue()); + assertEquals("Connection failures unexpected", 0, + nonTlsClient.getTestStatsProvider().getCounter(nameBuilder.toString() + + BookKeeperClientStats.FAILED_CONNECTION_COUNTER).get().longValue()); + + bookie.shutdown(); + assertEquals("Mismatch TLS channel count", 0, + tlsClient.getTestStatsProvider().getCounter(nameBuilder.toString() + + BookKeeperClientStats.ACTIVE_TLS_CHANNEL_COUNTER).get().longValue()); + assertEquals("Mismatch non-TLS channel count", 0, + nonTlsClient.getTestStatsProvider().getCounter(nameBuilder.toString() + + BookKeeperClientStats.ACTIVE_NON_TLS_CHANNEL_COUNTER).get().longValue()); + + } + } + + /** + * Verify handshake failure due to missing entry in trust store. + */ + @Test + public void testHandshakeFailure() throws Exception { + ClientConfiguration clientConf = new ClientConfiguration(baseClientConf) + .setNumChannelsPerBookie(1); + + // restart a bookie with wrong trust store + int restartBookieIdx = 0; + ServerConfiguration badBookieConf = bsConfs.get(restartBookieIdx); + + switch (serverTrustStoreFormat) { + case PEM: + badBookieConf.setTLSTrustStore(getResourcePath("server-cert.pem")); + break; + case JKS: + badBookieConf.setTLSTrustStore(getResourcePath("server-key.jks")) + .setTLSTrustStorePasswordPath(getResourcePath("keyStoreServerPassword.txt")); + break; + case PKCS12: + badBookieConf.setTLSTrustStore(getResourcePath("server-key.p12")) + .setTLSTrustStorePasswordPath(getResourcePath("keyStoreServerPassword.txt")); + break; + default: + throw new Exception("Unrecognized trust store format: " + serverTrustStoreFormat); + } + + + killBookie(restartBookieIdx); + LOG.info("Sleeping for 1s before restarting bookie with bad cert"); + Thread.sleep(1000); + BookieServer bookie = startBookie(badBookieConf); + bs.add(bookie); + bsConfs.add(badBookieConf); + + // Create ledger and write entries + TestStatsProvider testStatsProvider = new TestStatsProvider(); + BookKeeperTestClient client = new BookKeeperTestClient(clientConf, testStatsProvider); + byte[] passwd = "testPassword".getBytes(); + int numEntries = 2; + byte[] testEntry = "testEntry".getBytes(); + + // should fail to write entries whey WQ == AQ == 3 + try (LedgerHandle lh = client.createLedger(numBookies, numBookies, numBookies, DigestType.CRC32, passwd)) { + for (int i = 0; i <= numEntries; i++) { + lh.addEntry(testEntry); + } + fail("Should have failed with not enough bookies to write"); + } catch (BKException.BKNotEnoughBookiesException bke) { + // expected + } + + // check failed handshake counter + InetSocketAddress addr = bookie.getLocalAddress().getSocketAddress(); + StringBuilder nameBuilder = new StringBuilder(BookKeeperClientStats.CHANNEL_SCOPE) + .append(".") + .append(addr.getAddress().getHostAddress() + .replace('.', '_') + .replace('-', '_')) + .append("_") + .append(addr.getPort()) + .append("."); + + assertEquals("TLS handshake failure expected", 1, + client.getTestStatsProvider().getCounter(nameBuilder.toString() + + BookKeeperClientStats.FAILED_TLS_HANDSHAKE_COUNTER).get().longValue()); + } } -- To stop receiving notification emails like this one, please contact si...@apache.org.