HIVE-11909 : LLAP: merge master into branch (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/47187618 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/47187618 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/47187618 Branch: refs/heads/master Commit: 47187618bbdd014c04465ca3eb4c7843132c9c25 Parents: 7148ea0 2a65989 Author: Sergey Shelukhin <[email protected]> Authored: Mon Sep 21 16:04:08 2015 -0700 Committer: Sergey Shelukhin <[email protected]> Committed: Mon Sep 21 16:04:08 2015 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/log/PerfLogger.java | 196 +++++++++ .../HiveHBaseTableSnapshotInputFormat.java | 21 +- .../queries/positive/hbase_handler_snapshot.q | 4 + .../positive/hbase_handler_snapshot.q.out | 22 + .../hive/thrift/TestHadoop20SAuthBridge.java | 420 ------------------ .../hive/thrift/TestHadoopAuthBridge23.java | 423 +++++++++++++++++++ .../hive/metastore/RetryingHMSHandler.java | 33 +- .../java/org/apache/hadoop/hive/ql/Driver.java | 11 +- .../hadoop/hive/ql/exec/MapJoinOperator.java | 3 +- .../ql/exec/SparkHashTableSinkOperator.java | 3 +- .../apache/hadoop/hive/ql/exec/Utilities.java | 11 +- .../hadoop/hive/ql/exec/spark/SparkPlan.java | 3 +- .../hive/ql/exec/spark/SparkPlanGenerator.java | 9 +- .../hive/ql/exec/spark/SparkRecordHandler.java | 3 +- .../hadoop/hive/ql/exec/spark/SparkTask.java | 2 +- .../ql/exec/spark/status/SparkJobMonitor.java | 2 +- .../hive/ql/exec/tez/RecordProcessor.java | 4 +- .../hive/ql/exec/tez/ReduceRecordProcessor.java | 1 - .../hive/ql/exec/tez/ReduceRecordSource.java | 3 +- .../hadoop/hive/ql/exec/tez/TezJobMonitor.java | 17 +- .../hadoop/hive/ql/exec/tez/TezProcessor.java | 3 +- .../apache/hadoop/hive/ql/exec/tez/TezTask.java | 2 +- .../hive/ql/io/CombineHiveInputFormat.java | 10 +- .../hadoop/hive/ql/io/HiveInputFormat.java | 4 +- .../apache/hadoop/hive/ql/log/PerfLogger.java | 195 --------- .../hive/ql/optimizer/ppr/PartitionPruner.java | 7 +- .../hive/ql/parse/spark/SparkCompiler.java | 3 +- .../hadoop/hive/ql/session/SessionState.java | 37 +- .../persistence/TestBytesBytesMultiHashMap.java | 3 + .../ql/exec/persistence/TestHashPartition.java | 29 ++ .../clientpositive/parquet_ppd_boolean.q | 4 +- .../queries/clientpositive/parquet_ppd_char.q | 12 +- .../queries/clientpositive/parquet_ppd_date.q | 16 +- .../clientpositive/parquet_ppd_decimal.q | 32 +- .../clientpositive/parquet_ppd_timestamp.q | 16 +- .../clientpositive/parquet_ppd_varchar.q | 12 +- .../clientpositive/parquet_ppd_boolean.q.out | 28 +- .../clientpositive/parquet_ppd_char.q.out | 84 ++-- .../clientpositive/parquet_ppd_date.q.out | 112 ++--- .../clientpositive/parquet_ppd_decimal.q.out | 224 +++++----- .../clientpositive/parquet_ppd_timestamp.q.out | 112 ++--- .../clientpositive/parquet_ppd_varchar.q.out | 84 ++-- .../auth/LdapAuthenticationProviderImpl.java | 82 +++- .../hadoop/hive/shims/Hadoop20SShims.java | 5 +- .../apache/hadoop/hive/shims/Hadoop23Shims.java | 4 +- .../hive/thrift/HadoopThriftAuthBridge.java | 3 + 46 files changed, 1263 insertions(+), 1051 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/exec/SparkHashTableSinkOperator.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkPlanGenerator.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/RecordProcessor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java index f177f0d,91ba2bb..7c41cb6 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/ReduceRecordProcessor.java @@@ -53,10 -51,7 +53,9 @@@ import org.apache.tez.runtime.api.Logic import org.apache.tez.runtime.api.LogicalOutput; import org.apache.tez.runtime.api.ProcessorContext; import org.apache.tez.runtime.api.Reader; - import org.apache.tez.runtime.library.api.KeyValuesReader; +import com.google.common.collect.Lists; + /** * Process input from tez LogicalInput and write output - for a map plan * Just pump the records through the query plan. http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezJobMonitor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezProcessor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java index 3501d19,1ac1669..45ee9c5 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveInputFormat.java @@@ -36,11 -36,7 +36,10 @@@ import org.apache.hadoop.conf.Configura import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil; +import org.apache.hadoop.hive.llap.io.api.LlapIo; +import org.apache.hadoop.hive.llap.io.api.LlapIoProxy; - import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.exec.spark.SparkDynamicPartitionPruner; import org.apache.hadoop.hive.ql.plan.TableDesc; import org.apache.hadoop.hive.ql.exec.Operator; http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java ---------------------------------------------------------------------- diff --cc shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java index d47e3ff,93dcbd3..f60e8f0 --- a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java +++ b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java @@@ -68,11 -69,14 +68,13 @@@ import org.apache.hadoop.mapreduce.Task import org.apache.hadoop.mapreduce.TaskID; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.KerberosName; -import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; import org.apache.hadoop.tools.distcp2.DistCp; import org.apache.hadoop.tools.distcp2.DistCpOptions; + import org.apache.hadoop.tools.distcp2.DistCpOptions.FileAttribute; + import org.apache.hadoop.util.Progressable; -import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.VersionInfo; http://git-wip-us.apache.org/repos/asf/hive/blob/47187618/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java ---------------------------------------------------------------------- diff --cc shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java index fd6b17d,83369ee..5a136d8 --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java @@@ -95,11 -92,12 +95,12 @@@ import org.apache.hadoop.mapreduce.Task import org.apache.hadoop.mapreduce.task.JobContextImpl; import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; import org.apache.hadoop.tools.DistCp; import org.apache.hadoop.tools.DistCpOptions; + import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.tez.test.MiniTezCluster;
