HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/87083986 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/87083986 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/87083986 Branch: refs/heads/hive-14535 Commit: 870839862b500ec2dd6902573dc37bcacc665add Parents: 3e481b4 4340d46 Author: Sergey Shelukhin <[email protected]> Authored: Mon Sep 19 15:26:45 2016 -0700 Committer: Sergey Shelukhin <[email protected]> Committed: Mon Sep 19 15:26:45 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/hive/beeline/BeeLine.java | 5 +- .../org/apache/hadoop/hive/cli/CliDriver.java | 5 +- .../org/apache/hadoop/hive/conf/HiveConf.java | 13 +- .../hive/common/util/ShutdownHookManager.java | 171 +--- .../common/util/TestShutdownHookManager.java | 35 - dev-support/jenkins-common.sh | 63 ++ dev-support/jenkins-execute-build.sh | 157 ++-- .../hive/hbase/HiveHBaseTableInputFormat.java | 5 +- .../src/test/queries/positive/hbase_viewjoins.q | 56 ++ .../test/results/positive/hbase_viewjoins.q.out | 172 ++++ .../hive/hcatalog/common/HiveClientCache.java | 3 +- .../org/apache/hive/jdbc/TestJdbcDriver2.java | 806 ++++++++----------- .../test/resources/testconfiguration.properties | 87 +- .../apache/hive/jdbc/HiveDatabaseMetaData.java | 2 + .../hive/llap/security/SecretManager.java | 9 +- .../llap/security/SigningSecretManager.java | 4 +- .../hadoop/hive/metastore/HiveMetaStore.java | 53 +- .../hadoop/hive/metastore/MetaStoreUtils.java | 7 + .../apache/hadoop/hive/metastore/TableType.java | 2 +- .../org/apache/hadoop/hive/ql/ErrorMsg.java | 4 + .../apache/hadoop/hive/ql/QueryProperties.java | 15 + .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 102 ++- .../hadoop/hive/ql/exec/FunctionRegistry.java | 56 +- .../hive/ql/exec/mr/HadoopJobExecHelper.java | 3 +- .../spark/session/SparkSessionManagerImpl.java | 3 +- .../hadoop/hive/ql/exec/tez/TezJobMonitor.java | 3 +- .../ql/exec/vector/VectorDeserializeRow.java | 238 +++--- .../hive/ql/exec/vector/VectorMapOperator.java | 22 +- .../fast/VectorMapJoinFastLongHashTable.java | 2 +- .../fast/VectorMapJoinFastStringCommon.java | 2 +- .../VectorMapJoinOptimizedLongCommon.java | 56 -- .../VectorMapJoinOptimizedStringCommon.java | 26 - .../apache/hadoop/hive/ql/hooks/ATSHook.java | 3 +- .../zookeeper/CuratorFrameworkSingleton.java | 3 +- .../apache/hadoop/hive/ql/metadata/Table.java | 9 +- .../optimizer/SortedDynPartitionOptimizer.java | 9 +- .../hive/ql/optimizer/physical/Vectorizer.java | 25 +- .../hive/ql/parse/BaseSemanticAnalyzer.java | 16 +- .../hadoop/hive/ql/parse/CalcitePlanner.java | 63 +- .../hive/ql/parse/DDLSemanticAnalyzer.java | 16 +- .../apache/hadoop/hive/ql/parse/EximUtil.java | 2 +- .../apache/hadoop/hive/ql/parse/HiveParser.g | 29 + .../hive/ql/parse/ImportSemanticAnalyzer.java | 2 +- .../hive/ql/parse/LoadSemanticAnalyzer.java | 2 +- .../hadoop/hive/ql/parse/ParseContext.java | 27 +- .../org/apache/hadoop/hive/ql/parse/QB.java | 16 + .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 231 ++++-- .../hive/ql/parse/SemanticAnalyzerFactory.java | 3 + .../hadoop/hive/ql/parse/StorageFormat.java | 17 +- .../hadoop/hive/ql/parse/TaskCompiler.java | 106 ++- .../ql/parse/UpdateDeleteSemanticAnalyzer.java | 7 + .../hadoop/hive/ql/plan/CreateViewDesc.java | 93 ++- .../hadoop/hive/ql/plan/DropTableDesc.java | 27 +- .../hadoop/hive/ql/plan/HiveOperation.java | 3 + .../hadoop/hive/ql/plan/LoadFileDesc.java | 10 +- .../apache/hadoop/hive/ql/plan/PlanUtils.java | 41 + .../AuthorizationPreEventListener.java | 2 + .../authorization/plugin/HiveOperationType.java | 2 + .../plugin/sqlstd/Operation2Privilege.java | 6 + .../hive/ql/exec/TestFunctionRegistry.java | 15 +- .../ql/exec/vector/TestVectorRowObject.java | 2 +- .../hive/ql/exec/vector/TestVectorSerDeRow.java | 14 +- .../mapjoin/fast/CheckFastRowHashMap.java | 10 +- .../exec/vector/mapjoin/fast/VerifyFastRow.java | 2 +- ...ialized_view_authorization_create_no_grant.q | 15 + ...d_view_authorization_create_no_select_perm.q | 14 + ...materialized_view_authorization_drop_other.q | 14 + ...rialized_view_authorization_no_select_perm.q | 14 + .../clientnegative/materialized_view_delete.q | 10 + .../clientnegative/materialized_view_drop.q | 9 + .../clientnegative/materialized_view_drop2.q | 7 + .../clientnegative/materialized_view_insert.q | 6 + .../clientnegative/materialized_view_load.q | 7 + .../materialized_view_replace_with_view.q | 8 + .../clientnegative/materialized_view_update.q | 10 + .../test/queries/clientnegative/view_delete.q | 9 + .../test/queries/clientnegative/view_update.q | 9 + .../clientpositive/alter_partition_change_col.q | 2 +- .../clientpositive/alter_table_cascade.q | 4 +- .../clientpositive/authorization_view_sqlstd.q | 86 -- ql/src/test/queries/clientpositive/bucket5.q | 1 + .../dynpart_sort_optimization_acid.q | 13 + .../queries/clientpositive/groupby_sort_1_23.q | 4 +- .../clientpositive/groupby_sort_skew_1_23.q | 8 +- .../materialized_view_authorization_sqlstd.q | 56 ++ .../clientpositive/materialized_view_create.q | 31 + .../clientpositive/materialized_view_describe.q | 58 ++ .../clientpositive/materialized_view_drop.q | 7 + ql/src/test/queries/clientpositive/union32.q | 8 +- ql/src/test/queries/clientpositive/union33.q | 10 +- ql/src/test/queries/clientpositive/union36.q | 4 +- .../queries/clientpositive/unionDistinct_1.q | 16 +- .../queries/clientpositive/union_date_trim.q | 2 +- ql/src/test/queries/clientpositive/union_null.q | 8 +- .../queries/clientpositive/union_remove_12.q | 4 +- .../queries/clientpositive/union_remove_13.q | 4 +- .../queries/clientpositive/union_remove_14.q | 4 +- .../queries/clientpositive/union_type_chk.q | 7 - .../clientpositive/unionall_join_nullconstant.q | 2 +- .../clientpositive/view_authorization_sqlstd.q | 89 ++ ...zed_view_authorization_create_no_grant.q.out | 26 + ...ew_authorization_create_no_select_perm.q.out | 20 + ...rialized_view_authorization_drop_other.q.out | 30 + ...ized_view_authorization_no_select_perm.q.out | 30 + .../materialized_view_delete.q.out | 19 + .../clientnegative/materialized_view_drop.q.out | 42 + .../materialized_view_drop2.q.out | 34 + .../materialized_view_insert.q.out | 19 + .../clientnegative/materialized_view_load.q.out | 19 + .../materialized_view_replace_with_view.q.out | 30 + .../materialized_view_update.q.out | 19 + .../results/clientnegative/view_delete.q.out | 19 + .../results/clientnegative/view_update.q.out | 19 + .../alter_partition_change_col.q.out | 4 +- .../clientpositive/alter_table_cascade.q.out | 8 +- .../authorization_view_sqlstd.q.out | 269 ------- .../dynpart_sort_optimization_acid.q.out | 267 +++++- .../clientpositive/groupby_sort_1_23.q.out | 122 +-- .../clientpositive/groupby_sort_skew_1_23.q.out | 132 +-- .../results/clientpositive/llap/bucket5.q.out | 586 ++++++++++++++ .../results/clientpositive/llap/bucket6.q.out | 203 +++++ .../clientpositive/llap/bucket_many.q.out | 223 +++++ .../llap/bucketizedhiveinputformat.q.out | 112 +++ .../clientpositive/llap/bucketmapjoin6.q.out | 146 ++++ .../clientpositive/llap/bucketmapjoin7.q.out | 315 ++++++++ .../llap/empty_dir_in_table.q.out | 46 ++ .../llap/exchgpartition2lel.q.out | 203 +++++ ...rnal_table_with_space_in_location_path.q.out | 88 ++ .../llap/file_with_header_footer.q.out | 184 +++++ .../llap/import_exported_table.q.out | 28 + .../llap/infer_bucket_sort_bucketed_table.q.out | 121 +++ .../clientpositive/llap/input16_cc.q.out | 534 ++++++++++++ .../clientpositive/llap/insert_dir_distcp.q.out | 14 + .../llap/join_acid_non_acid.q.out | 58 ++ .../clientpositive/llap/leftsemijoin_mr.q.out | 98 +++ .../llap/list_bucket_dml_10.q.out | 256 ++++++ .../results/clientpositive/llap/load_fs2.q.out | 121 +++ .../load_hdfs_file_with_space_in_the_name.q.out | 25 + .../llap/non_native_window_udf.q.out | 52 ++ .../clientpositive/llap/quotedid_smb.q.out | 81 ++ .../llap/reduce_deduplicate.q.out | 387 +++++++++ .../clientpositive/llap/remote_script.q.out | 49 ++ .../clientpositive/llap/schemeAuthority.q.out | 84 ++ .../clientpositive/llap/schemeAuthority2.q.out | 53 ++ .../llap/table_nonprintable.q.out | 72 ++ .../llap/temp_table_external.q.out | 34 + .../clientpositive/llap/uber_reduce.q.out | 31 + .../clientpositive/llap/unionDistinct_1.q.out | 70 +- .../clientpositive/llap/union_type_chk.q.out | 30 - ...materialized_view_authorization_sqlstd.q.out | 185 +++++ .../materialized_view_create.q.out | 170 ++++ .../materialized_view_describe.q.out | 335 ++++++++ .../clientpositive/materialized_view_drop.q.out | 39 + .../spark/groupby_sort_1_23.q.out | 134 +-- .../spark/groupby_sort_skew_1_23.q.out | 144 ++-- .../results/clientpositive/spark/union32.q.out | 130 +-- .../results/clientpositive/spark/union33.q.out | 16 +- .../clientpositive/spark/union_date_trim.q.out | 4 +- .../clientpositive/spark/union_null.q.out | 12 +- .../clientpositive/spark/union_remove_12.q.out | 48 +- .../clientpositive/spark/union_remove_13.q.out | 52 +- .../clientpositive/spark/union_remove_14.q.out | 48 +- .../test/results/clientpositive/union32.q.out | 130 +-- .../test/results/clientpositive/union33.q.out | 16 +- .../test/results/clientpositive/union36.q.out | 28 +- .../clientpositive/unionDistinct_1.q.out | 70 +- .../clientpositive/union_date_trim.q.out | 4 +- .../results/clientpositive/union_null.q.out | 12 +- .../clientpositive/union_remove_12.q.out | 8 +- .../clientpositive/union_remove_13.q.out | 26 +- .../clientpositive/union_remove_14.q.out | 8 +- .../results/clientpositive/union_type_chk.q.out | 30 - .../unionall_join_nullconstant.q.out | 4 +- .../view_authorization_sqlstd.q.out | 295 +++++++ .../fast/BinarySortableDeserializeRead.java | 132 ++- .../hive/serde2/fast/DeserializeRead.java | 71 +- .../lazy/fast/LazySimpleDeserializeRead.java | 770 +++++++++--------- .../fast/LazyBinaryDeserializeRead.java | 119 +-- .../hive/serde2/typeinfo/TypeInfoUtils.java | 3 + .../apache/hadoop/hive/serde2/VerifyFast.java | 2 +- .../binarysortable/TestBinarySortableFast.java | 35 +- .../hive/serde2/lazy/TestLazySimpleFast.java | 31 +- .../serde2/lazybinary/TestLazyBinaryFast.java | 32 +- .../cli/operation/ClassicTableTypeMapping.java | 5 + .../apache/hive/service/server/HiveServer2.java | 3 +- testutils/ptest2/pom.xml | 41 +- .../hive/ptest/execution/ExecutionPhase.java | 8 + .../hive/ptest/execution/HostExecutor.java | 7 +- .../org/apache/hive/ptest/execution/PTest.java | 3 +- .../org/apache/hive/ptest/execution/Phase.java | 2 +- .../ptest/execution/conf/FileListProvider.java | 31 + .../ptest/execution/conf/QFileTestBatch.java | 20 +- .../hive/ptest/execution/conf/TestBatch.java | 30 +- .../ptest/execution/conf/TestConfiguration.java | 4 + .../hive/ptest/execution/conf/TestParser.java | 71 +- .../ptest/execution/conf/UnitTestBatch.java | 91 ++- .../conf/UnitTestPropertiesParser.java | 666 +++++++++++++++ .../ptest2/src/main/resources/batch-exec.vm | 10 +- .../ptest2/src/main/resources/source-prep.vm | 20 +- .../execution/MockRSyncCommandExecutor.java | 7 + .../ptest/execution/MockSSHCommandExecutor.java | 7 + .../ptest/execution/TestExecutionPhase.java | 10 +- ...ExecutionPhase.testFailingQFile.approved.txt | 4 +- ...cutionPhase.testFailingUnitTest.approved.txt | 10 +- ...utionPhase.testPassingQFileTest.approved.txt | 4 +- ...cutionPhase.testPassingUnitTest.approved.txt | 10 +- .../hive/ptest/execution/TestHostExecutor.java | 52 +- .../TestHostExecutor.testBasic.approved.txt | 24 +- ...xecutor.testIsolatedFailsOnExec.approved.txt | 10 +- ...tor.testIsolatedFailsOnRsyncOne.approved.txt | 8 +- ...testIsolatedFailsOnRsyncUnknown.approved.txt | 8 +- ...xecutor.testParallelFailsOnExec.approved.txt | 10 +- ...ecutor.testParallelFailsOnRsync.approved.txt | 8 +- ...tScripts.testAlternativeTestJVM.approved.txt | 16 +- .../TestScripts.testBatch.approved.txt | 16 +- .../TestScripts.testPrepGit.approved.txt | 13 +- .../TestScripts.testPrepHadoop1.approved.txt | 20 +- .../TestScripts.testPrepNone.approved.txt | 13 +- .../TestScripts.testPrepSvn.approved.txt | 13 +- .../execution/conf/TestQFileTestBatch.java | 9 +- .../ptest/execution/conf/TestTestParser.java | 10 +- .../conf/TestUnitTestPropertiesParser.java | 672 ++++++++++++++++ .../ptest2/src/test/resources/log4j2.properties | 62 ++ .../resources/test-configuration2.properties | 154 ++++ 224 files changed, 11098 insertions(+), 3219 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/87083986/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/87083986/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/87083986/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/87083986/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/87083986/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/87083986/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/87083986/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/87083986/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadFileDesc.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/87083986/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java index f055cde,b15ad34..a07237e --- a/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java @@@ -380,7 -380,62 +380,48 @@@ public final class PlanUtils return ret; } + /** + * Generate a table descriptor from a createViewDesc. + */ + public static TableDesc getTableDesc(CreateViewDesc crtViewDesc, String cols, String colTypes) { + TableDesc ret; + + try { + Class serdeClass = JavaUtils.loadClass(crtViewDesc.getSerde()); + ret = getTableDesc(serdeClass, new String(LazySerDeParameters.DefaultSeparators), cols, + colTypes, false, false); + + // set other table properties + /* + TODO - I don't think I need any of this + Properties properties = ret.getProperties(); + + if (crtTblDesc.getTableName() != null && crtTblDesc.getDatabaseName() != null) { + properties.setProperty(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_NAME, + crtTblDesc.getTableName()); + } + + if (crtTblDesc.getTblProps() != null) { + properties.putAll(crtTblDesc.getTblProps()); + } + */ + + // replace the default input & output file format with those found in + // crtTblDesc + Class<? extends InputFormat> inClass = + (Class<? extends InputFormat>)JavaUtils.loadClass(crtViewDesc.getInputFormat()); + Class<? extends HiveOutputFormat> outClass = + (Class<? extends HiveOutputFormat>)JavaUtils.loadClass(crtViewDesc.getOutputFormat()); + + ret.setInputFileFormatClass(inClass); + ret.setOutputFileFormatClass(outClass); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Unable to find class in getTableDesc: " + e.getMessage(), e); + } + return ret; + } + /** - * Generate the table descriptor of MetadataTypedColumnsetSerDe with the - * separatorCode. MetaDataTypedColumnsetSerDe is used because LazySimpleSerDe - * does not support a table with a single column "col" with type - * "array<string>". - */ - public static TableDesc getDefaultTableDesc(String separatorCode) { - return new TableDesc( - TextInputFormat.class, IgnoreKeyTextOutputFormat.class, Utilities - .makeProperties( - org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT,separatorCode, - serdeConstants.SERIALIZATION_LIB,MetadataTypedColumnsetSerDe.class.getName())); - } - - /** * Generate the table descriptor for reduce key. */ public static TableDesc getReduceKeyTableDesc(List<FieldSchema> fieldSchemas,
