HIVE-14023. LLAP: Make the Hive query id available in ContainerRunner. (Siddharth Seth, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c5b4d66d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c5b4d66d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c5b4d66d Branch: refs/heads/master Commit: c5b4d66d572fe0f589a2da56ed95560ed287ae73 Parents: c168af2 Author: Siddharth Seth <ss...@apache.org> Authored: Tue Jun 21 10:48:48 2016 -0700 Committer: Siddharth Seth <ss...@apache.org> Committed: Tue Jun 21 10:48:48 2016 -0700 ---------------------------------------------------------------------- .../ext/LlapTaskUmbilicalExternalClient.java | 17 +- .../daemon/rpc/LlapDaemonProtocolProtos.java | 2503 +++++++----------- .../apache/hadoop/hive/llap/tez/Converters.java | 35 +- .../src/protobuf/LlapDaemonProtocol.proto | 37 +- .../hadoop/hive/llap/tez/TestConverters.java | 23 +- .../hadoop/hive/llap/LlapBaseInputFormat.java | 6 +- .../llap/daemon/impl/ContainerRunnerImpl.java | 60 +- .../hive/llap/daemon/impl/QueryTracker.java | 12 +- .../llap/daemon/impl/TaskRunnerCallable.java | 6 +- .../daemon/impl/TaskExecutorTestHelpers.java | 14 +- .../TestFirstInFirstOutComparator.java | 30 +- .../llap/tezplugins/LlapTaskCommunicator.java | 57 +- .../tezplugins/helpers/SourceStateTracker.java | 9 +- .../tezplugins/TestLlapTaskCommunicator.java | 12 + .../ql/udf/generic/GenericUDTFGetSplits.java | 19 +- 15 files changed, 1124 insertions(+), 1716 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c5b4d66d/llap-client/src/java/org/apache/hadoop/hive/llap/ext/LlapTaskUmbilicalExternalClient.java ---------------------------------------------------------------------- diff --git a/llap-client/src/java/org/apache/hadoop/hive/llap/ext/LlapTaskUmbilicalExternalClient.java b/llap-client/src/java/org/apache/hadoop/hive/llap/ext/LlapTaskUmbilicalExternalClient.java index 5f250b4..4933fb3 100644 --- a/llap-client/src/java/org/apache/hadoop/hive/llap/ext/LlapTaskUmbilicalExternalClient.java +++ b/llap-client/src/java/org/apache/hadoop/hive/llap/ext/LlapTaskUmbilicalExternalClient.java @@ -34,10 +34,11 @@ import com.google.protobuf.InvalidProtocolBufferException; import org.apache.commons.collections4.ListUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos; +import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.QueryIdentifierProto; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SignableVertexSpec; +import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmissionStateProto; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkRequestProto; -import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.VertexIdentifier; +import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.SubmitWorkResponseProto; import org.apache.hadoop.hive.llap.daemon.rpc.LlapDaemonProtocolProtos.VertexOrBinary; import org.apache.hadoop.hive.llap.protocol.LlapTaskUmbilicalProtocol; import org.apache.hadoop.hive.llap.security.LlapTokenIdentifier; @@ -155,9 +156,9 @@ public class LlapTaskUmbilicalExternalClient extends AbstractService implements } catch (InvalidProtocolBufferException e) { throw new RuntimeException(e); } - VertexIdentifier vId = vertex.getVertexIdentifier(); - TezTaskAttemptID attemptId = Converters.createTaskAttemptId( - vId, request.getFragmentNumber(), request.getAttemptNumber()); + QueryIdentifierProto queryIdentifierProto = vertex.getQueryIdentifier(); + TezTaskAttemptID attemptId = Converters.createTaskAttemptId(queryIdentifierProto, + vertex.getVertexIndex(), request.getFragmentNumber(), request.getAttemptNumber()); final String fragmentId = attemptId.toString(); pendingEvents.putIfAbsent(fragmentId, new PendingEventData( @@ -169,12 +170,12 @@ public class LlapTaskUmbilicalExternalClient extends AbstractService implements // Send out the actual SubmitWorkRequest communicator.sendSubmitWork(request, llapHost, llapPort, - new LlapProtocolClientProxy.ExecuteRequestCallback<LlapDaemonProtocolProtos.SubmitWorkResponseProto>() { + new LlapProtocolClientProxy.ExecuteRequestCallback<SubmitWorkResponseProto>() { @Override - public void setResponse(LlapDaemonProtocolProtos.SubmitWorkResponseProto response) { + public void setResponse(SubmitWorkResponseProto response) { if (response.hasSubmissionState()) { - if (response.getSubmissionState().equals(LlapDaemonProtocolProtos.SubmissionStateProto.REJECTED)) { + if (response.getSubmissionState().equals(SubmissionStateProto.REJECTED)) { String msg = "Fragment: " + fragmentId + " rejected. Server Busy."; LOG.info(msg); if (responder != null) {
