HIVE-13118 : add some logging to LLAP token related paths (Sergey Shelukhin, reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1c2d7ba3 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1c2d7ba3 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1c2d7ba3 Branch: refs/heads/master Commit: 1c2d7ba35d5e357e125490e63d9ccc84dbff263c Parents: 9f61fc6 Author: Sergey Shelukhin <ser...@apache.org> Authored: Tue Feb 23 11:30:22 2016 -0800 Committer: Sergey Shelukhin <ser...@apache.org> Committed: Tue Feb 23 11:30:22 2016 -0800 ---------------------------------------------------------------------- .../hive/llap/tez/LlapProtocolClientProxy.java | 6 +++++ .../hive/llap/security/LlapTokenIdentifier.java | 2 +- .../daemon/impl/LlapProtocolServerImpl.java | 3 +++ .../hive/llap/security/LlapSecurityHelper.java | 23 ++++++++++++-------- .../llap/tezplugins/LlapTaskCommunicator.java | 3 +++ .../hive/ql/exec/tez/TezSessionState.java | 4 +++- 6 files changed, 30 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/1c2d7ba3/llap-client/src/java/org/apache/hadoop/hive/llap/tez/LlapProtocolClientProxy.java ---------------------------------------------------------------------- diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/tez/LlapProtocolClientProxy.java b/llap-client/src/java/org/apache/hadoop/hive/llap/tez/LlapProtocolClientProxy.java index 5b0674a..e8d4148 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/tez/LlapProtocolClientProxy.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/tez/LlapProtocolClientProxy.java @@ -472,6 +472,9 @@ public class LlapProtocolClientProxy extends AbstractService { LlapProtocolBlockingPB proxy = hostProxies.get(hostId); if (proxy == null) { if (llapToken == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Creating a client without a token for " + nodeId); + } proxy = new LlapProtocolClientImpl(getConfig(), nodeId.getHostname(), nodeId.getPort(), retryPolicy, socketFactory); } else { @@ -485,6 +488,9 @@ public class LlapProtocolClientProxy extends AbstractService { SecurityUtil.setTokenService(nodeToken, NetUtils.createSocketAddrForHost( nodeId.getHostname(), nodeId.getPort())); ugi.addToken(nodeToken); + if (LOG.isDebugEnabled()) { + LOG.debug("Creating a client for " + nodeId + "; the token is " + nodeToken); + } proxy = ugi.doAs(new PrivilegedAction<LlapProtocolBlockingPB>() { @Override public LlapProtocolBlockingPB run() { http://git-wip-us.apache.org/repos/asf/hive/blob/1c2d7ba3/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java ---------------------------------------------------------------------- diff --git a/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java index c4bee75..23980d0 100644 --- a/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java +++ b/llap-common/src/java/org/apache/hadoop/hive/llap/security/LlapTokenIdentifier.java @@ -71,7 +71,7 @@ public class LlapTokenIdentifier extends AbstractDelegationTokenIdentifier { @Override public String toString() { - return KIND; + return KIND + "; " + super.toString(); } @InterfaceAudience.Private http://git-wip-us.apache.org/repos/asf/hive/blob/1c2d7ba3/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java index c386d77..3a25a66 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapProtocolServerImpl.java @@ -238,6 +238,9 @@ public class LlapProtocolServerImpl extends AbstractService LlapTokenIdentifier llapId = new LlapTokenIdentifier(owner, renewer, realUser); // TODO: note that the token is not renewable right now and will last for 2 weeks by default. Token<LlapTokenIdentifier> token = new Token<LlapTokenIdentifier>(llapId, zkSecretManager); + if (LOG.isInfoEnabled()) { + LOG.info("Created LLAP token " + token); + } ByteArrayDataOutput out = ByteStreams.newDataOutput(); try { token.write(out); http://git-wip-us.apache.org/repos/asf/hive/blob/1c2d7ba3/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java index aa8745d..6d8b2da 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/security/LlapSecurityHelper.java @@ -54,6 +54,7 @@ public class LlapSecurityHelper implements LlapTokenProvider { private ServiceInstanceSet activeInstances; private final Configuration conf; private LlapManagementProtocolClientImpl client; + private ServiceInstance clientInstance; private final SocketFactory socketFactory; private final RetryPolicy retryPolicy; @@ -87,23 +88,24 @@ public class LlapSecurityHelper implements LlapTokenProvider { // We could have also added keytab support; right now client must do smth like kinit. } Iterator<ServiceInstance> llaps = null; - ServiceInstance someLlap = null; - if (client == null) { + if (clientInstance == null) { + assert client == null; llaps = getLlapServices(false); - someLlap = llaps.next(); + clientInstance = llaps.next(); } ByteString tokenBytes = null; boolean hasRefreshed = false; while (true) { try { - tokenBytes = getTokenBytes(someLlap); + tokenBytes = getTokenBytes(); break; } catch (InterruptedException ie) { throw new RuntimeException(ie); } catch (IOException ex) { LOG.error("Cannot get a token, trying a different instance", ex); client = null; + clientInstance = null; } if (llaps == null || !llaps.hasNext()) { if (hasRefreshed) { // Only refresh once. @@ -112,7 +114,7 @@ public class LlapSecurityHelper implements LlapTokenProvider { llaps = getLlapServices(true); hasRefreshed = true; } - someLlap = llaps.next(); + clientInstance = llaps.next(); } // Stupid protobuf byte-buffer reinvention. @@ -120,17 +122,20 @@ public class LlapSecurityHelper implements LlapTokenProvider { DataInputByteBuffer in = new DataInputByteBuffer(); in.reset(tokenBytes.asReadOnlyByteBuffer()); token.readFields(in); + if (LOG.isInfoEnabled()) { + LOG.info("Obtained a LLAP delegation token from " + clientInstance + ": " + token); + } return token; } - private ByteString getTokenBytes( - final ServiceInstance si) throws InterruptedException, IOException { + private ByteString getTokenBytes() throws InterruptedException, IOException { return llapUgi.doAs(new PrivilegedExceptionAction<ByteString>() { @Override public ByteString run() throws Exception { + assert clientInstance != null; if (client == null) { - client = new LlapManagementProtocolClientImpl( - conf, si.getHost(), si.getManagementPort(), retryPolicy, socketFactory); + client = new LlapManagementProtocolClientImpl(conf, clientInstance.getHost(), + clientInstance.getManagementPort(), retryPolicy, socketFactory); } // Client only connects on the first call, so this has to be done in doAs. GetTokenRequestProto req = GetTokenRequestProto.newBuilder().build(); http://git-wip-us.apache.org/repos/asf/hive/blob/1c2d7ba3/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java ---------------------------------------------------------------------- diff --git a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java index 76d095a..456121b 100644 --- a/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java +++ b/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java @@ -123,6 +123,9 @@ public class LlapTaskCommunicator extends TezTaskCommunicatorImpl { } else { this.token = null; } + if (LOG.isInfoEnabled()) { + LOG.info("Task communicator with a token " + token); + } Preconditions.checkState((token != null) == UserGroupInformation.isSecurityEnabled()); umbilical = new LlapTaskUmbilicalProtocolImpl(getUmbilical()); http://git-wip-us.apache.org/repos/asf/hive/blob/1c2d7ba3/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java index 5f9235b..6675f0d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java @@ -276,7 +276,9 @@ public class TezSessionState { if (UserGroupInformation.isSecurityEnabled()) { LlapTokenProvider tp = LlapProxy.getOrInitTokenProvider(conf); Token<LlapTokenIdentifier> token = tp.getDelegationToken(); - LOG.info("Obtained a token: " + token); + if (LOG.isInfoEnabled()) { + LOG.info("Obtained a LLAP token: " + token); + } llapCredentials = new Credentials(); llapCredentials.addToken(LlapTokenIdentifier.KIND_NAME, token); }
