KUDU-2262: allow Java client to retry if no master is a leader Currently, when authentication is required, the Java client will not retry if encounters NOT_AUTHORIZED error. This causes a request to fail in cases when masters are in the process of the very first leader election after started up, even if the client has valid authentication token. Since the non-leader masters only have self-signed cert.
This patch updates the logic to throw a recoverable exception if client has valid secondary authn credentials (such as authn token), but it does not have primary authn credentials (such as Kerberos creds), and retry the request as long as the original call has not timed out. Corresponding test is added. Change-Id: Ia39a8d77cbf58c6f2f1f97eaf5e2e17ac1fa09fa Reviewed-on: http://gerrit.cloudera.org:8080/9088 Tested-by: Kudu Jenkins Reviewed-by: Todd Lipcon <[email protected]> Reviewed-by: Alexey Serbin <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f62e4cd0 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f62e4cd0 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f62e4cd0 Branch: refs/heads/master Commit: f62e4cd0e19cdb3f9dc66d0809cc178e9569b3fa Parents: a0d348a Author: hahao <[email protected]> Authored: Fri Jan 19 17:05:36 2018 -0800 Committer: Hao Hao <[email protected]> Committed: Thu Jan 25 00:17:42 2018 +0000 ---------------------------------------------------------------------- .../java/org/apache/kudu/client/Negotiator.java | 15 ++++++++++-- .../org/apache/kudu/client/TestSecurity.java | 25 ++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/f62e4cd0/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java b/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java index 029c516..466f265 100644 --- a/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java +++ b/java/kudu-client/src/main/java/org/apache/kudu/client/Negotiator.java @@ -345,7 +345,7 @@ public class Negotiator extends SimpleChannelUpstreamHandler { } } - private void chooseAndInitializeSaslMech(NegotiatePB response) throws NonRecoverableException { + private void chooseAndInitializeSaslMech(NegotiatePB response) throws KuduException { // Gather the set of server-supported mechanisms. Map<String, String> errorsByMech = Maps.newHashMap(); Set<SaslMechanism> serverMechs = Sets.newHashSet(); @@ -419,7 +419,18 @@ public class Negotiator extends SimpleChannelUpstreamHandler { message = "client/server supported SASL mechanism mismatch: [" + Joiner.on(", ").withKeyValueSeparator(": ").join(errorsByMech) + "]"; } - throw new NonRecoverableException(Status.NotAuthorized(message)); + + // If client has valid secondary authn credentials (such as authn token), + // but it does not have primary authn credentials (such as Kerberos creds), + // throw a recoverable exception. So that the request can be retried as long + // as the original call hasn't timed out, for cases documented in KUDU-2267, + // e.g. masters are in the process of the very first leader election after + // started up and does not have CA signed cert. + if (authnToken != null) { + throw new RecoverableException(Status.NotAuthorized(message)); + } else { + throw new NonRecoverableException(Status.NotAuthorized(message)); + } } private AuthenticationTypePB.TypeCase chooseAuthenticationType(NegotiatePB response) { http://git-wip-us.apache.org/repos/asf/kudu/blob/f62e4cd0/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java ---------------------------------------------------------------------- diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java index d2f8fcb..c9881c4 100644 --- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java +++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestSecurity.java @@ -28,6 +28,7 @@ public class TestSecurity extends BaseKuduTest { @BeforeClass public static void setUpBeforeClass() throws Exception { miniClusterBuilder.enableKerberos() + .addMasterFlag("--leader_failure_max_missed_heartbeat_periods=10.0") .addMasterFlag("--rpc_trace_negotiation"); BaseKuduTest.setUpBeforeClass(); @@ -70,4 +71,28 @@ public class TestSecurity extends BaseKuduTest { System.setProperty(SecurityUtil.KUDU_TICKETCACHE_PROPERTY, oldTicketCache); } } + + /** + * Test that a client is able to connect to masters using valid tokens + * after all masters were killed and restarted, and before a leader is + * elected. Leader election time is configured to be long enough using + * '--leader_failure_max_missed_heartbeat_periods'. + */ + @Test + public void testConnectToNonLeaderMasters() throws Exception { + byte[] authnData = client.exportAuthenticationCredentials().join(); + assertNotNull(authnData); + String oldTicketCache = System.getProperty(SecurityUtil.KUDU_TICKETCACHE_PROPERTY); + System.clearProperty(SecurityUtil.KUDU_TICKETCACHE_PROPERTY); + try { + KuduClient newClient = new KuduClient.KuduClientBuilder(masterAddresses).build(); + newClient.importAuthenticationCredentials(authnData); + + miniCluster.killMasters(); + miniCluster.restartDeadMasters(); + newClient.listTabletServers(); + } finally { + System.setProperty(SecurityUtil.KUDU_TICKETCACHE_PROPERTY, oldTicketCache); + } + } }
