Merge branch 'master' into llap
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e073cce2 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e073cce2 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e073cce2 Branch: refs/heads/llap Commit: e073cce2354801e12c4767f47a76fbf25168d026 Parents: 342668f e9a7218 Author: Jason Dere <[email protected]> Authored: Mon May 2 18:29:03 2016 -0700 Committer: Jason Dere <[email protected]> Committed: Mon May 2 18:29:03 2016 -0700 ---------------------------------------------------------------------- .../apache/hive/beeline/DatabaseConnection.java | 12 +- .../org/apache/hadoop/hive/cli/CliDriver.java | 5 +- .../org/apache/hadoop/hive/conf/HiveConf.java | 19 + data/files/keystore.jks | Bin 2248 -> 2206 bytes data/files/keystore_exampledotcom.jks | Bin 0 -> 2222 bytes data/files/struct1_a.txt | 4 + data/files/struct1_b.txt | 1 + data/files/struct1_c.txt | 1 + data/files/struct2_a.txt | 2 + data/files/struct2_b.txt | 2 + data/files/struct2_c.txt | 1 + data/files/struct2_d.txt | 1 + data/files/struct3_a.txt | 2 + data/files/struct3_b.txt | 1 + data/files/struct3_c.txt | 1 + data/files/struct4_a.txt | 2 + data/files/struct4_b.txt | 1 + data/files/struct4_c.txt | 1 + data/files/truststore.jks | Bin 958 -> 1816 bytes .../hcatalog/templeton/ExecServiceImpl.java | 7 +- .../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 35 +- .../org/apache/hive/jdbc/TestJdbcDriver2.java | 25 +- .../test/java/org/apache/hive/jdbc/TestSSL.java | 71 +- itests/pom.xml | 3 - .../test/resources/testconfiguration.properties | 34 +- .../org/apache/hive/jdbc/HiveConnection.java | 99 +- .../java/org/apache/hive/jdbc/JdbcColumn.java | 13 +- .../hadoop/hive/llap/cli/LlapServiceDriver.java | 21 +- .../llap/cli/LlapStatusOptionsProcessor.java | 44 +- .../hive/llap/cli/LlapStatusServiceDriver.java | 74 +- .../daemon/impl/LlapProtocolServerImpl.java | 57 +- .../hive/llap/io/api/impl/LlapInputFormat.java | 2 +- .../main/resources/llap-cli-log4j2.properties | 2 +- llap-server/src/main/resources/package.py | 7 +- packaging/src/main/assembly/bin.xml | 5 + .../hive/ql/exec/AbstractMapOperator.java | 178 + .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 7 + .../hadoop/hive/ql/exec/FunctionRegistry.java | 8 + .../apache/hadoop/hive/ql/exec/MapOperator.java | 86 +- .../apache/hadoop/hive/ql/exec/Utilities.java | 33 +- .../hadoop/hive/ql/exec/mr/ExecMapper.java | 3 +- .../ql/exec/spark/SparkMapRecordHandler.java | 3 +- .../hadoop/hive/ql/exec/tez/DagUtils.java | 3 +- .../hive/ql/exec/tez/MapRecordProcessor.java | 15 +- .../hive/ql/exec/tez/MapRecordSource.java | 6 +- .../hive/ql/exec/tez/ReduceRecordSource.java | 4 +- .../vector/VectorAppMasterEventOperator.java | 16 +- .../hive/ql/exec/vector/VectorAssignRow.java | 1111 ++-- .../ql/exec/vector/VectorAssignRowDynBatch.java | 41 - .../exec/vector/VectorAssignRowSameBatch.java | 36 - .../ql/exec/vector/VectorDeserializeRow.java | 1114 ++-- .../hive/ql/exec/vector/VectorExtractRow.java | 971 +--- .../exec/vector/VectorExtractRowDynBatch.java | 40 - .../exec/vector/VectorExtractRowSameBatch.java | 36 - .../ql/exec/vector/VectorFileSinkOperator.java | 16 +- .../ql/exec/vector/VectorGroupByOperator.java | 13 +- .../exec/vector/VectorMapJoinBaseOperator.java | 11 +- .../ql/exec/vector/VectorMapJoinOperator.java | 4 +- .../VectorMapJoinOuterFilteredOperator.java | 17 +- .../hive/ql/exec/vector/VectorMapOperator.java | 848 ++- .../exec/vector/VectorReduceSinkOperator.java | 16 +- .../exec/vector/VectorSMBMapJoinOperator.java | 11 +- .../VectorSparkHashTableSinkOperator.java | 16 +- ...VectorSparkPartitionPruningSinkOperator.java | 13 +- .../ql/exec/vector/VectorizationContext.java | 12 +- .../ql/exec/vector/VectorizedBatchUtil.java | 49 + .../VectorMapJoinGenerateResultOperator.java | 8 +- .../fast/VectorMapJoinFastLongHashUtil.java | 10 +- .../fast/VectorMapJoinFastStringCommon.java | 10 +- .../hadoop/hive/ql/io/HiveInputFormat.java | 6 +- .../hadoop/hive/ql/io/NullRowsInputFormat.java | 2 +- .../hadoop/hive/ql/io/orc/OrcInputFormat.java | 4 +- .../ql/io/parquet/MapredParquetInputFormat.java | 2 +- .../hive/ql/optimizer/physical/Vectorizer.java | 334 +- .../hadoop/hive/ql/parse/FromClauseParser.g | 4 +- .../hadoop/hive/ql/parse/IdentifiersParser.g | 6 +- .../hadoop/hive/ql/parse/SelectClauseParser.g | 4 +- .../apache/hadoop/hive/ql/plan/BaseWork.java | 23 + .../org/apache/hadoop/hive/ql/plan/MapWork.java | 11 + .../org/apache/hadoop/hive/ql/plan/TezWork.java | 9 +- .../hive/ql/plan/VectorPartitionConversion.java | 172 +- .../hive/ql/plan/VectorPartitionDesc.java | 164 +- .../hadoop/hive/ql/udf/generic/BaseMaskUDF.java | 473 ++ .../hive/ql/udf/generic/GenericUDFMask.java | 334 ++ .../ql/udf/generic/GenericUDFMaskFirstN.java | 229 + .../hive/ql/udf/generic/GenericUDFMaskHash.java | 77 + .../ql/udf/generic/GenericUDFMaskLastN.java | 193 + .../udf/generic/GenericUDFMaskShowFirstN.java | 248 + .../ql/udf/generic/GenericUDFMaskShowLastN.java | 198 + .../ql/exec/vector/TestVectorRowObject.java | 14 +- .../hive/ql/exec/vector/TestVectorSerDeRow.java | 169 +- .../hive/ql/io/orc/TestInputOutputFormat.java | 10 +- .../hive/ql/plan/TestTezWorkConcurrency.java | 65 + .../avro_schema_evolution_native.q | 18 + .../queries/clientpositive/bucket_groupby.q | 33 +- .../queries/clientpositive/groupby_sort_10.q | 2 + .../schema_evol_orc_acidvec_mapwork_part.q | 3 + .../schema_evol_orc_acidvec_mapwork_table.q | 3 + .../schema_evol_orc_nonvec_mapwork_table.q | 2 - .../schema_evol_orc_vec_mapwork_part.q | 3 + .../schema_evol_orc_vec_mapwork_table.q | 7 +- .../schema_evol_text_fetchwork_table.q | 56 - .../schema_evol_text_mapwork_table.q | 56 - .../schema_evol_text_nonvec_fetchwork_part.q | 98 - .../schema_evol_text_nonvec_fetchwork_table.q | 67 - .../schema_evol_text_nonvec_mapwork_part.q | 828 ++- ..._evol_text_nonvec_mapwork_part_all_complex.q | 159 + ...vol_text_nonvec_mapwork_part_all_primitive.q | 509 ++ .../schema_evol_text_nonvec_mapwork_table.q | 822 ++- .../schema_evol_text_vec_mapwork_part.q | 827 +++ ...ema_evol_text_vec_mapwork_part_all_complex.q | 164 + ...a_evol_text_vec_mapwork_part_all_primitive.q | 514 ++ .../schema_evol_text_vec_mapwork_table.q | 826 +++ .../schema_evol_text_vecrow_mapwork_part.q | 827 +++ ..._evol_text_vecrow_mapwork_part_all_complex.q | 165 + ...vol_text_vecrow_mapwork_part_all_primitive.q | 514 ++ .../schema_evol_text_vecrow_mapwork_table.q | 826 +++ .../clientpositive/tez_schema_evolution.q | 1 + ql/src/test/queries/clientpositive/udf_mask.q | 13 + .../queries/clientpositive/udf_mask_first_n.q | 13 + .../test/queries/clientpositive/udf_mask_hash.q | 13 + .../queries/clientpositive/udf_mask_last_n.q | 13 + .../clientpositive/udf_mask_show_first_n.q | 13 + .../clientpositive/udf_mask_show_last_n.q | 13 + .../beelinepositive/show_functions.q.out | 6 + .../results/clientnegative/subq_insert.q.out | 2 +- .../avro_schema_evolution_native.q.out | 206 + .../results/clientpositive/bucket_groupby.q.out | 308 +- .../results/clientpositive/groupby1_limit.q.out | 2 +- .../clientpositive/groupby_sort_10.q.out | 8 +- .../results/clientpositive/regexp_extract.q.out | 137 - .../schema_evol_text_fetchwork_table.q.out | 298 -- .../schema_evol_text_mapwork_table.q.out | 298 -- ...schema_evol_text_nonvec_fetchwork_part.q.out | 642 --- ...chema_evol_text_nonvec_fetchwork_table.q.out | 297 -- .../schema_evol_text_nonvec_mapwork_part.q.out | 4909 ++++++++++++++++-- ...l_text_nonvec_mapwork_part_all_complex.q.out | 726 +++ ...text_nonvec_mapwork_part_all_primitive.q.out | 3038 +++++++++++ .../schema_evol_text_nonvec_mapwork_table.q.out | 4376 +++++++++++++++- .../schema_evol_text_vec_mapwork_part.q.out | 4479 ++++++++++++++++ ...evol_text_vec_mapwork_part_all_complex.q.out | 730 +++ ...ol_text_vec_mapwork_part_all_primitive.q.out | 3058 +++++++++++ .../schema_evol_text_vec_mapwork_table.q.out | 4221 +++++++++++++++ .../schema_evol_text_vecrow_mapwork_part.q.out | 4479 ++++++++++++++++ ...l_text_vecrow_mapwork_part_all_complex.q.out | 732 +++ ...text_vecrow_mapwork_part_all_primitive.q.out | 3058 +++++++++++ .../schema_evol_text_vecrow_mapwork_table.q.out | 4221 +++++++++++++++ .../results/clientpositive/show_functions.q.out | 12 + .../tez/schema_evol_text_fetchwork_table.q.out | 298 -- .../tez/schema_evol_text_mapwork_table.q.out | 298 -- ...schema_evol_text_nonvec_fetchwork_part.q.out | 642 --- ...chema_evol_text_nonvec_fetchwork_table.q.out | 297 -- .../schema_evol_text_nonvec_mapwork_part.q.out | 4453 ++++++++++++++-- ...l_text_nonvec_mapwork_part_all_complex.q.out | 669 +++ ...text_nonvec_mapwork_part_all_primitive.q.out | 2734 ++++++++++ .../schema_evol_text_nonvec_mapwork_table.q.out | 3920 +++++++++++++- .../tez/schema_evol_text_vec_mapwork_part.q.out | 3999 ++++++++++++++ ...evol_text_vec_mapwork_part_all_complex.q.out | 673 +++ ...ol_text_vec_mapwork_part_all_primitive.q.out | 2738 ++++++++++ .../schema_evol_text_vec_mapwork_table.q.out | 3741 +++++++++++++ .../schema_evol_text_vecrow_mapwork_part.q.out | 3999 ++++++++++++++ ...l_text_vecrow_mapwork_part_all_complex.q.out | 675 +++ ...text_vecrow_mapwork_part_all_primitive.q.out | 2738 ++++++++++ .../schema_evol_text_vecrow_mapwork_table.q.out | 3741 +++++++++++++ .../vector_orc_string_reader_empty_dict.q.out | 62 + .../tez/vector_partition_diff_num_cols.q.out | 1 + .../tez/vector_tablesample_rows.q.out | 307 ++ .../test/results/clientpositive/udf_mask.q.out | 71 + .../clientpositive/udf_mask_first_n.q.out | 68 + .../results/clientpositive/udf_mask_hash.q.out | 59 + .../clientpositive/udf_mask_last_n.q.out | 68 + .../clientpositive/udf_mask_show_first_n.q.out | 68 + .../clientpositive/udf_mask_show_last_n.q.out | 68 + .../vector_partition_diff_num_cols.q.out | 1 + .../vector_tablesample_rows.q.out | 2 - .../compiler/errors/missing_overwrite.q.out | 2 +- .../fast/BinarySortableDeserializeRead.java | 806 +-- .../hive/serde2/fast/DeserializeRead.java | 379 +- .../lazy/fast/LazySimpleDeserializeRead.java | 704 +-- .../fast/LazyBinaryDeserializeRead.java | 944 +--- .../apache/hadoop/hive/serde2/VerifyFast.java | 75 +- .../hive/service/auth/HiveAuthFactory.java | 21 +- 182 files changed, 84895 insertions(+), 10148 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/e073cce2/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --cc common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 7db492f,caadf2a..5f78ae6 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@@ -2658,13 -2670,17 +2670,20 @@@ public class HiveConf extends Configura "ZooKeeper for ZooKeeper SecretManager."), LLAP_ZKSM_ZK_CONNECTION_STRING("hive.llap.zk.sm.connectionString", "", "ZooKeeper connection string for ZooKeeper SecretManager."), + LLAP_ZK_REGISTRY_USER("hive.llap.zk.registry.user", "", + "In the LLAP ZooKeeper-based registry, specifies the username in the Zookeeper path.\n" + + "This should be the hive user or whichever user is running the LLAP daemon."), // Note: do not rename to ..service.acl; Hadoop generates .hosts setting name from this, // resulting in a collision with existing hive.llap.daemon.service.hosts and bizarre errors. + // These are read by Hadoop IPC, so you should check the usage and naming conventions (e.g. + // ".blocked" is a string hardcoded by Hadoop, and defaults are enforced elsewhere in Hive) + // before making changes or copy-pasting these. LLAP_SECURITY_ACL("hive.llap.daemon.acl", "*", "The ACL for LLAP daemon."), + LLAP_SECURITY_ACL_DENY("hive.llap.daemon.acl.blocked", "", "The deny ACL for LLAP daemon."), LLAP_MANAGEMENT_ACL("hive.llap.management.acl", "*", "The ACL for LLAP daemon management."), + LLAP_MANAGEMENT_ACL_DENY("hive.llap.management.acl.blocked", "", + "The deny ACL for LLAP daemon management."), + // Hadoop DelegationTokenManager default is 1 week. LLAP_DELEGATION_TOKEN_LIFETIME("hive.llap.daemon.delegation.token.lifetime", "14d", new TimeValidator(TimeUnit.SECONDS), http://git-wip-us.apache.org/repos/asf/hive/blob/e073cce2/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/e073cce2/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java index 7a3d6a6,9a9f43a..dc63d7b --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java @@@ -33,9 -33,9 +33,10 @@@ import org.slf4j.Logger import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; + import org.apache.hadoop.hive.ql.exec.AbstractMapOperator; import org.apache.hadoop.hive.llap.io.api.LlapProxy; import org.apache.hadoop.hive.ql.CompilationOpContext; +import org.apache.hadoop.hive.llap.LlapOutputFormat; import org.apache.hadoop.hive.ql.exec.DummyStoreOperator; import org.apache.hadoop.hive.ql.exec.HashTableDummyOperator; import org.apache.hadoop.hive.ql.exec.MapOperator;
