HIVE-HIVE-13419 part 2. Merge remote-tracking branch 'origin/master' into llap
Conflicts: llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/79c1c691 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/79c1c691 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/79c1c691 Branch: refs/heads/llap Commit: 79c1c691e529107cba27eabefb94707645a39987 Parents: a7b0ca7 91ab819 Author: Siddharth Seth <ss...@apache.org> Authored: Mon Apr 4 15:37:10 2016 -0700 Committer: Siddharth Seth <ss...@apache.org> Committed: Mon Apr 4 15:37:10 2016 -0700 ---------------------------------------------------------------------- beeline/pom.xml.orig | 169 + .../java/org/apache/hive/beeline/BeeLine.java | 18 +- .../apache/hive/beeline/cli/TestHiveCli.java | 15 +- data/conf/llap/hive-site.xml | 44 + data/conf/llap/llap-daemon-site.xml | 61 - .../org/apache/hive/minikdc/MiniHiveKdc.java | 46 +- .../minikdc/TestJdbcNonKrbSASLWithMiniKdc.java | 103 + itests/hive-unit/pom.xml | 13 +- .../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 26 +- .../test/resources/testconfiguration.properties | 7 - itests/util/pom.xml | 11 + .../apache/hadoop/hive/llap/LlapItUtils.java | 90 + .../org/apache/hadoop/hive/ql/QTestUtil.java | 25 +- .../impl/LlapZookeeperRegistryImpl.java | 9 +- .../hive/llap/tez/LlapProtocolClientProxy.java | 4 +- .../hive/llap/cli/LlapOptionsProcessor.java | 25 +- .../configuration/LlapDaemonConfiguration.java | 8 + .../hive/llap/daemon/impl/LlapDaemon.java | 34 +- .../daemon/impl/LlapProtocolServerImpl.java | 7 +- .../daemon/services/impl/LlapWebServices.java | 13 +- .../hive/llap/daemon/MiniLlapCluster.java | 147 +- .../llap/tezplugins/LlapTaskCommunicator.java | 37 +- .../tezplugins/LlapTaskSchedulerService.java | 7 + .../tezplugins/helpers/SourceStateTracker.java | 2 +- .../tezplugins/TestLlapTaskCommunicator.java | 304 +- .../java/org/apache/orc/impl/IntegerReader.java | 4 +- .../apache/orc/impl/RunLengthIntegerReader.java | 7 +- .../orc/impl/RunLengthIntegerReaderV2.java | 7 +- .../org/apache/orc/impl/SerializationUtils.java | 34 +- .../apache/orc/impl/TestSerializationUtils.java | 45 +- .../hadoop/hive/ql/exec/FunctionRegistry.java | 9 +- .../ql/exec/spark/HiveSparkClientFactory.java | 9 + .../ql/exec/spark/RemoteHiveSparkClient.java | 20 +- .../hadoop/hive/ql/exec/spark/SparkTask.java | 3 + .../exec/spark/status/LocalSparkJobMonitor.java | 2 +- .../spark/status/RemoteSparkJobMonitor.java | 5 +- .../ql/exec/vector/VectorizationContext.java | 12 +- .../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 16 +- .../hive/ql/io/orc/TreeReaderFactory.java | 46 +- .../hadoop/hive/ql/parse/ParseContext.java | 7 + .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 29 +- .../hadoop/hive/ql/parse/TaskCompiler.java | 2 +- .../hive/ql/plan/ExprNodeGenericFuncDesc.java | 5 +- .../hive/ql/exec/TestFunctionRegistry.java | 18 +- .../exec/vector/TestVectorizationContext.java | 17 +- .../queries/clientpositive/cast_on_constant.q | 7 + ql/src/test/queries/clientpositive/macro_1.q | 29 + .../clientpositive/cast_on_constant.q.out | 160 + .../test/results/clientpositive/macro_1.q.out | 84 + .../clientpositive/spark/temp_table.q.out | 2 + service-rpc/src/gen/thrift/gen-py/__init__.py | 0 service/src/gen/thrift/gen-cpp/ThriftHive.cpp | 3544 ++++++++ service/src/gen/thrift/gen-cpp/ThriftHive.h | 1224 +++ .../gen-cpp/ThriftHive_server.skeleton.cpp | 84 + .../thrift/gen-cpp/hive_service_constants.cpp | 17 + .../gen/thrift/gen-cpp/hive_service_constants.h | 24 + .../gen/thrift/gen-cpp/hive_service_types.cpp | 351 + .../src/gen/thrift/gen-cpp/hive_service_types.h | 176 + .../hadoop/hive/service/HiveClusterStatus.java | 901 ++ .../hive/service/HiveServerException.java | 601 ++ .../hadoop/hive/service/JobTrackerState.java | 45 + .../apache/hadoop/hive/service/ThriftHive.java | 7784 ++++++++++++++++++ service/src/gen/thrift/gen-php/ThriftHive.php | 1943 +++++ service/src/gen/thrift/gen-php/Types.php | 338 + service/src/gen/thrift/gen-py/__init__.py | 0 .../gen-py/hive_service/ThriftHive-remote | 1242 +++ .../thrift/gen-py/hive_service/ThriftHive.py | 1674 ++++ .../gen/thrift/gen-py/hive_service/__init__.py | 1 + .../gen/thrift/gen-py/hive_service/constants.py | 11 + .../gen/thrift/gen-py/hive_service/ttypes.py | 260 + .../gen/thrift/gen-rb/hive_service_constants.rb | 9 + .../src/gen/thrift/gen-rb/hive_service_types.rb | 68 + service/src/gen/thrift/gen-rb/thrift_hive.rb | 555 ++ .../hive/service/auth/HiveAuthFactory.java | 13 +- .../service/cli/thrift/ThriftCLIService.java | 15 +- .../apache/hadoop/hive/shims/Hadoop23Shims.java | 66 +- .../apache/hadoop/hive/shims/HadoopShims.java | 2 +- .../hive/spark/client/SparkClientImpl.java | 9 +- 78 files changed, 22374 insertions(+), 387 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/79c1c691/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/79c1c691/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/79c1c691/llap-client/src/java/org/apache/hadoop/hive/llap/registry/impl/LlapZookeeperRegistryImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/79c1c691/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java ---------------------------------------------------------------------- diff --cc llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java index e9213ee,2fe59a2..418d87a --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java @@@ -290,9 -292,23 +293,24 @@@ public class LlapDaemon extends Composi ShuffleHandler.initializeAndStart(shuffleHandlerConf); LOG.info("Setting shuffle port to: " + ShuffleHandler.get().getPort()); this.shufflePort.set(ShuffleHandler.get().getPort()); + getConfig() + .setInt(ConfVars.LLAP_DAEMON_YARN_SHUFFLE_PORT.varname, ShuffleHandler.get().getPort()); super.serviceStart(); + + // Setup the actual ports in the configuration. + getConfig().setInt(ConfVars.LLAP_DAEMON_RPC_PORT.varname, server.getBindAddress().getPort()); + getConfig().setInt(ConfVars.LLAP_MANAGEMENT_RPC_PORT.varname, server.getManagementBindAddress().getPort()); + if (webServices != null) { + getConfig().setInt(ConfVars.LLAP_DAEMON_WEB_PORT.varname, webServices.getPort()); + } + LlapOutputFormatService.get(); - LOG.info("LlapDaemon serviceStart complete"); + + this.registry.init(getConfig()); + this.registry.start(); + LOG.info( + "LlapDaemon serviceStart complete. RPC Port={}, ManagementPort={}, ShuflePort={}, WebPort={}", + server.getBindAddress().getPort(), server.getManagementBindAddress().getPort(), + ShuffleHandler.get().getPort(), (webServices == null ? "" : webServices.getPort())); } public void serviceStop() throws Exception { http://git-wip-us.apache.org/repos/asf/hive/blob/79c1c691/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskCommunicator.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/79c1c691/llap-tez/src/java/org/apache/hadoop/hive/llap/tezplugins/LlapTaskSchedulerService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/79c1c691/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java ----------------------------------------------------------------------