HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6d914483 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6d914483 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6d914483 Branch: refs/heads/hive-14535 Commit: 6d9144835cbdda27fea172f0fb268f4f4aac1a0c Parents: 70299dc 297b443 Author: Sergey Shelukhin <[email protected]> Authored: Fri Sep 30 18:07:43 2016 -0700 Committer: Sergey Shelukhin <[email protected]> Committed: Fri Sep 30 18:07:43 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/hive/beeline/BeeLine.java | 20 +- .../org/apache/hive/beeline/BeeLineOpts.java | 2 +- beeline/src/main/resources/BeeLine.properties | 11 +- .../beeline/TestBeeLineExceptionHandling.java | 72 ++ .../hive/common/metrics/LegacyMetrics.java | 96 +-- .../hive/common/metrics/MetricsMBean.java | 13 +- .../hive/common/metrics/MetricsMBeanImpl.java | 16 +- .../hive/common/metrics/common/Metrics.java | 31 +- .../common/metrics/common/MetricsConstant.java | 7 + .../metrics/metrics2/CodahaleMetrics.java | 70 +- .../org/apache/hadoop/hive/conf/HiveConf.java | 14 +- .../apache/hadoop/hive/ql/log/PerfLogger.java | 33 +- .../hive/common/metrics/TestLegacyMetrics.java | 103 +-- data/conf/llap/tez-site.xml | 12 + data/conf/tez/hive-site.xml | 10 - dev-support/jenkins-execute-build.sh | 30 +- druid-handler/pom.xml | 11 +- .../src/test/queries/positive/hbase_bulk.m | 62 -- .../src/test/queries/positive/hbase_bulk.q | 62 ++ .../src/test/results/positive/hbase_bulk.m.out | 133 ---- .../src/test/results/positive/hbase_bulk.q.out | 133 ++++ .../hcatalog/streaming/StreamingConnection.java | 2 + .../hcatalog/streaming/TransactionBatch.java | 8 +- .../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 4 + .../apache/hadoop/hive/metastore/FakeDerby.java | 424 ++++++++++++ .../metastore/TestObjectStoreInitRetry.java | 127 ++++ .../TestPartitionNameWhitelistValidation.java | 53 +- .../org/apache/hive/jdbc/TestJdbcDriver2.java | 32 + itests/qtest-spark/pom.xml | 2 + itests/qtest/pom.xml | 2 + .../hive/cli/TestHBaseMinimrCliDriver.java | 62 -- .../test/resources/testconfiguration.properties | 2 - .../hive/cli/control/AbstractCliConfig.java | 23 +- .../hadoop/hive/cli/control/CliConfigs.java | 38 +- .../hadoop/hive/cli/control/CoreCliDriver.java | 3 +- .../org/apache/hadoop/hive/ql/QTestUtil.java | 344 +++++++--- .../org/apache/hive/jdbc/HiveConnection.java | 79 +++ jdbc/src/java/org/apache/hive/jdbc/Utils.java | 1 + .../org/apache/hive/jdbc/TestJdbcDriver.java | 98 +++ .../hive/llap/registry/ServiceInstanceSet.java | 7 +- .../registry/impl/InactiveServiceInstance.java | 77 +++ .../registry/impl/LlapFixedRegistryImpl.java | 2 +- .../impl/LlapZookeeperRegistryImpl.java | 34 +- .../org/apache/hadoop/hive/llap/LlapUtil.java | 12 + .../hadoop/hive/llap/LlapBaseInputFormat.java | 6 +- .../hive/llap/daemon/impl/LlapDaemon.java | 5 +- .../daemon/services/impl/LlapWebServices.java | 2 +- .../llap/tezplugins/LlapTaskCommunicator.java | 14 +- .../tezplugins/LlapTaskSchedulerService.java | 2 +- .../tezplugins/TestLlapTaskCommunicator.java | 5 + .../hive/metastore/HMSMetricsListener.java | 52 +- .../hadoop/hive/metastore/HiveMetaStore.java | 13 +- .../hadoop/hive/metastore/ObjectStore.java | 71 ++ pom.xml | 15 +- ql/pom.xml | 26 +- .../java/org/apache/hadoop/hive/ql/Driver.java | 18 +- .../hadoop/hive/ql/exec/FileSinkOperator.java | 10 +- .../hadoop/hive/ql/exec/FunctionInfo.java | 20 +- .../hadoop/hive/ql/exec/FunctionRegistry.java | 4 +- .../hadoop/hive/ql/exec/ReduceSinkOperator.java | 9 + .../apache/hadoop/hive/ql/exec/Registry.java | 96 ++- .../org/apache/hadoop/hive/ql/exec/Task.java | 9 + .../hadoop/hive/ql/exec/WindowFunctionInfo.java | 4 +- .../hadoop/hive/ql/exec/mr/MapRedTask.java | 7 + .../hadoop/hive/ql/exec/mr/MapredLocalTask.java | 7 + .../exec/spark/HiveBaseFunctionResultList.java | 96 ++- .../hive/ql/exec/spark/HiveMapFunction.java | 2 +- .../hive/ql/exec/spark/HiveReduceFunction.java | 2 +- .../hive/ql/exec/spark/SortByShuffler.java | 84 ++- .../hadoop/hive/ql/exec/spark/SparkTask.java | 7 + .../spark/status/impl/JobMetricsListener.java | 4 +- .../tez/HostAffinitySplitLocationProvider.java | 80 ++- .../apache/hadoop/hive/ql/exec/tez/TezTask.java | 7 + .../apache/hadoop/hive/ql/exec/tez/Utils.java | 8 +- .../org/apache/hadoop/hive/ql/io/AcidUtils.java | 6 - .../hadoop/hive/ql/lockmgr/DbTxnManager.java | 3 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 2 +- .../optimizer/SortedDynPartitionOptimizer.java | 4 +- .../translator/SqlFunctionConverter.java | 2 +- .../org/apache/hadoop/hive/ql/parse/HiveLexer.g | 4 + .../hadoop/hive/ql/parse/IdentifiersParser.g | 57 +- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +- .../apache/hadoop/hive/ql/udf/UDFDateFloor.java | 8 +- .../ql/udf/generic/GenericUDFLoggedInUser.java | 82 +++ .../hive/ql/exec/TestFunctionRegistry.java | 100 +++ .../hadoop/hive/ql/exec/mr/TestMapRedTask.java | 47 ++ .../hive/ql/exec/mr/TestMapredLocalTask.java | 46 ++ .../ql/exec/spark/TestHiveKVResultCache.java | 5 +- .../hive/ql/exec/spark/TestSparkTask.java | 46 ++ .../TestHostAffinitySplitLocationProvider.java | 150 ++++- .../hadoop/hive/ql/exec/tez/TestTezTask.java | 17 + .../hadoop/hive/ql/plan/TestViewEntity.java | 31 + .../ql/udf/TestUDFDateFormatGranularity.java | 22 +- .../mapreduce_stack_trace_hadoop20.q | 13 - .../mapreduce_stack_trace_turnoff_hadoop20.q | 13 - ql/src/test/queries/clientpositive/extract.q | 43 ++ ql/src/test/queries/clientpositive/floor_time.q | 46 ++ .../queries/clientpositive/udf_logged_in_user.q | 5 + .../mapreduce_stack_trace_hadoop20.q.out | 13 - ...mapreduce_stack_trace_turnoff_hadoop20.q.out | 5 - .../results/clientpositive/cbo_union_view.q.out | 12 + .../dynpart_sort_optimization_acid.q.out | 20 +- .../test/results/clientpositive/extract.q.out | 200 ++++++ .../results/clientpositive/floor_time.q.out | 211 ++++++ .../results/clientpositive/ppd_union_view.q.out | 12 + .../results/clientpositive/show_functions.q.out | 5 + .../clientpositive/udf_logged_in_user.q.out | 22 + .../results/clientpositive/union_view.q.out | 54 ++ .../apache/hadoop/hive/serde2/SerDeUtils.java | 11 + service/pom.xml | 7 + .../auth/LdapAuthenticationProviderImpl.java | 657 ++----------------- .../service/auth/ldap/ChainFilterFactory.java | 78 +++ .../auth/ldap/CustomQueryFilterFactory.java | 84 +++ .../hive/service/auth/ldap/DirSearch.java | 52 ++ .../service/auth/ldap/DirSearchFactory.java | 37 ++ .../apache/hive/service/auth/ldap/Filter.java | 36 + .../hive/service/auth/ldap/FilterFactory.java | 33 + .../service/auth/ldap/GroupFilterFactory.java | 90 +++ .../hive/service/auth/ldap/LdapSearch.java | 155 +++++ .../service/auth/ldap/LdapSearchFactory.java | 64 ++ .../hive/service/auth/ldap/LdapUtils.java | 228 +++++++ .../apache/hive/service/auth/ldap/Query.java | 154 +++++ .../hive/service/auth/ldap/QueryFactory.java | 135 ++++ .../service/auth/ldap/SearchResultHandler.java | 163 +++++ .../service/auth/ldap/UserFilterFactory.java | 75 +++ .../auth/ldap/UserSearchFilterFactory.java | 65 ++ .../hive/service/cli/operation/Operation.java | 33 +- .../service/cli/operation/SQLOperation.java | 18 +- .../service/cli/session/SessionManager.java | 33 +- .../apache/hive/service/server/HiveServer2.java | 7 +- .../auth/TestLdapAtnProviderWithMiniDS.java | 3 +- .../TestLdapAuthenticationProviderImpl.java | 277 +++++++- .../hive/service/auth/ldap/Credentials.java | 41 ++ .../hive/service/auth/ldap/LdapTestUtils.java | 126 ++++ .../hive/service/auth/ldap/TestChainFilter.java | 103 +++ .../auth/ldap/TestCustomQueryFilter.java | 85 +++ .../hive/service/auth/ldap/TestGroupFilter.java | 101 +++ .../hive/service/auth/ldap/TestLdapSearch.java | 209 ++++++ .../hive/service/auth/ldap/TestLdapUtils.java | 103 +++ .../hive/service/auth/ldap/TestQuery.java | 59 ++ .../service/auth/ldap/TestQueryFactory.java | 79 +++ .../auth/ldap/TestSearchResultHandler.java | 222 +++++++ .../hive/service/auth/ldap/TestUserFilter.java | 75 +++ .../service/auth/ldap/TestUserSearchFilter.java | 94 +++ .../apache/hadoop/hive/shims/Hadoop23Shims.java | 73 ++- .../apache/hadoop/hive/shims/HadoopShims.java | 2 + spark-client/pom.xml | 15 +- .../hive/spark/client/MetricsCollection.java | 8 +- .../apache/hive/spark/client/RemoteDriver.java | 4 +- .../hive/spark/client/SparkClientImpl.java | 8 + .../hive/spark/client/metrics/InputMetrics.java | 9 +- .../hive/spark/client/metrics/Metrics.java | 6 +- .../client/metrics/ShuffleReadMetrics.java | 18 +- .../client/metrics/ShuffleWriteMetrics.java | 4 +- .../hive/spark/client/rpc/RpcConfiguration.java | 42 +- .../apache/hive/spark/client/rpc/RpcServer.java | 44 +- .../spark/client/TestMetricsCollection.java | 8 +- .../apache/hive/spark/client/rpc/TestRpc.java | 66 +- .../ptest2/conf/cloudhost.properties.example | 37 ++ testutils/ptest2/pom.xml | 5 + .../execution/context/CloudComputeService.java | 224 +++++-- .../context/CloudExecutionContextProvider.java | 105 ++- .../resources/test-configuration2.properties | 3 +- 163 files changed, 7018 insertions(+), 1825 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/6d914483/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/6d914483/itests/qtest-spark/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/6d914483/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/6d914483/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 32e4daf,74bde3d..6db28a5 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@@ -25,9 -25,8 +25,10 @@@ import java.lang.reflect.Field import java.net.InetAddress; import java.net.URI; import java.nio.ByteBuffer; +import java.sql.Connection; +import java.sql.SQLException; import java.util.ArrayList; + import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; http://git-wip-us.apache.org/repos/asf/hive/blob/6d914483/ql/src/java/org/apache/hadoop/hive/ql/Driver.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/6d914483/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/6d914483/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/6d914483/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/6d914483/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ----------------------------------------------------------------------
