HIVE-14671 : merge master into hive-14535 (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/42335b46 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/42335b46 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/42335b46 Branch: refs/heads/hive-14535 Commit: 42335b46cacb2ebcc4569e1081a1cc6369558146 Parents: 14c9482 c254557 Author: sergey <[email protected]> Authored: Mon Oct 9 17:37:32 2017 -0700 Committer: sergey <[email protected]> Committed: Mon Oct 9 17:37:32 2017 -0700 ---------------------------------------------------------------------- .../mr/AccumuloIndexedOutputFormat.java | 50 +- .../positive/accumulo_predicate_pushdown.q.out | 2 +- .../accumulo_single_sourced_multi_insert.q.out | 4 +- .../java/org/apache/hive/beeline/BeeLine.java | 110 +- .../apache/hive/beeline/ClassNameCompleter.java | 254 -- .../org/apache/hive/beeline/HiveSchemaTool.java | 36 +- .../hive/beeline/TestBeelineArgParsing.java | 13 +- .../hive/beeline/TestClassNameCompleter.java | 76 - bin/ext/tokentool.sh | 67 + .../hive/common/ValidCompactorTxnList.java | 89 - .../hadoop/hive/common/ValidReadTxnList.java | 237 -- .../apache/hadoop/hive/common/ValidTxnList.java | 112 - .../hadoop/hive/common/log/LogRedirector.java | 99 + .../org/apache/hadoop/hive/conf/HiveConf.java | 62 +- .../apache/hive/common/util/HiveTestUtils.java | 37 +- .../java/org/apache/hive/http/HttpServer.java | 34 +- .../hive/common/TestValidReadTxnList.java | 109 - data/conf/spark/local/hive-site.xml | 258 ++ data/files/birthdays | 9 + data/scripts/q_test_init.sql | 10 +- .../results/positive/external_table_ppd.q.out | 1 - .../positive/hbase_binary_storage_queries.q.out | 2 - .../src/test/results/positive/hbase_ddl.q.out | 2 - .../src/test/results/positive/hbasestats.q.out | 5 - .../apache/hive/hcatalog/data/JsonSerDe.java | 15 +- .../mapreduce/FileOutputCommitterContainer.java | 11 +- .../hive/hcatalog/mapreduce/Security.java | 3 + .../hive/hcatalog/streaming/TestStreaming.java | 4 +- .../streaming/mutate/StreamingTestUtils.java | 4 +- .../hive/hcatalog/api/TestHCatClient.java | 2 +- .../svr/src/main/config/webhcat-default.xml | 8 + .../hive/hcatalog/templeton/AppConfig.java | 1 + .../apache/hive/hcatalog/templeton/Main.java | 43 + .../test/queries/clientpositive/multiple_agg.q | 9 + ...rt_addpartition_blobstore_to_blobstore.q.out | 4 - ...import_addpartition_blobstore_to_local.q.out | 4 - ...rt_addpartition_blobstore_to_warehouse.q.out | 4 - ...import_addpartition_local_to_blobstore.q.out | 4 - .../import_blobstore_to_blobstore.q.out | 2 - .../import_blobstore_to_local.q.out | 2 - .../import_blobstore_to_warehouse.q.out | 2 - .../import_local_to_blobstore.q.out | 2 - .../results/clientpositive/multiple_agg.q.out | 35 + .../hive/metastore/TestHiveMetaStoreTxns.java | 4 +- .../apache/hadoop/hive/ql/TestAcidOnTez.java | 5 +- .../hadoop/hive/ql/history/TestHiveHistory.java | 4 +- .../hive/ql/parse/TestReplicationScenarios.java | 63 +- .../hive/ql/txn/compactor/TestCompactor.java | 4 +- .../hive/cli/TestLocalSparkCliDriver.java | 62 + .../test/resources/testconfiguration.properties | 2 + .../hadoop/hive/cli/control/CliConfigs.java | 22 + .../hive/cli/control/CoreBeeLineDriver.java | 41 +- .../java/org/apache/hive/beeline/QFile.java | 124 +- .../apache/hive/beeline/QFileBeeLineClient.java | 138 +- .../llap/io/decode/OrcEncodedDataConsumer.java | 7 +- .../llap/io/encoded/SerDeEncodedDataReader.java | 15 +- .../hive/llap/cache/TestLowLevelCacheImpl.java | 4 +- .../upgrade/derby/046-HIVE-17566.derby.sql | 25 + .../upgrade/derby/hive-schema-3.0.0.derby.sql | 41 +- .../derby/upgrade-2.3.0-to-3.0.0.derby.sql | 1 + .../upgrade/hive/hive-schema-3.0.0.hive.sql | 17 + .../upgrade/mssql/031-HIVE-17566.mssql.sql | 76 + .../upgrade/mssql/hive-schema-3.0.0.mssql.sql | 78 +- .../mssql/upgrade-2.3.0-to-3.0.0.mssql.sql | 1 + .../upgrade/mysql/045-HIVE-16886.mysql.sql | 2 +- .../upgrade/mysql/046-HIVE-17566.mysql.sql | 57 + .../upgrade/mysql/hive-schema-3.0.0.mysql.sql | 64 +- .../mysql/upgrade-2.3.0-to-3.0.0.mysql.sql | 1 + .../upgrade/oracle/046-HIVE-17566.oracle.sql | 76 + .../upgrade/oracle/hive-schema-3.0.0.oracle.sql | 78 +- .../oracle/upgrade-2.3.0-to-3.0.0.oracle.sql | 1 + .../postgres/045-HIVE-17566.postgres.sql | 88 + .../postgres/hive-schema-3.0.0.postgres.sql | 91 + .../upgrade-2.3.0-to-3.0.0.postgres.sql | 1 + .../hadoop/hive/metastore/HiveMetaStore.java | 14 + .../hive/metastore/HiveMetaStoreClient.java | 2 + .../hive/metastore/MetaStoreDirectSql.java | 2 + .../hadoop/hive/metastore/ObjectStore.java | 2 +- .../hadoop/hive/metastore/cache/CacheUtils.java | 8 +- .../hive/metastore/cache/CachedStore.java | 545 ++- .../hive/metastore/cache/SharedCache.java | 124 +- .../datasource/BoneCPDataSourceProvider.java | 95 - .../datasource/DataSourceProvider.java | 79 - .../datasource/DataSourceProviderFactory.java | 47 - .../datasource/HikariCPDataSourceProvider.java | 97 - .../hive/metastore/datasource/package-info.java | 23 - .../hive/metastore/tools/SQLGenerator.java | 172 - .../hive/metastore/txn/CompactionInfo.java | 151 - .../metastore/txn/CompactionTxnHandler.java | 960 ----- .../hadoop/hive/metastore/txn/TxnDbUtil.java | 369 -- .../hadoop/hive/metastore/txn/TxnHandler.java | 3667 ------------------ .../hadoop/hive/metastore/txn/TxnStore.java | 433 --- .../hadoop/hive/metastore/txn/TxnUtils.java | 242 -- .../hive/metastore/cache/TestCachedStore.java | 113 +- .../TestDataSourceProviderFactory.java | 173 - .../metastore/txn/TestTxnHandlerNegative.java | 54 - .../hadoop/hive/metastore/txn/TestTxnUtils.java | 215 - .../txn/TestValidCompactorTxnList.java | 136 - pom.xml | 2 + .../hadoop/hive/llap/LlapCacheAwareFs.java | 4 +- .../java/org/apache/hadoop/hive/ql/Driver.java | 4 +- .../org/apache/hadoop/hive/ql/ErrorMsg.java | 35 +- .../apache/hadoop/hive/ql/QueryProperties.java | 13 +- .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 20 +- .../apache/hadoop/hive/ql/exec/MoveTask.java | 9 +- .../hadoop/hive/ql/exec/ReplCopyTask.java | 18 +- .../hadoop/hive/ql/exec/StatsNoJobTask.java | 2 +- .../apache/hadoop/hive/ql/exec/StatsTask.java | 9 +- .../apache/hadoop/hive/ql/exec/TaskFactory.java | 4 - .../apache/hadoop/hive/ql/exec/Utilities.java | 34 +- .../hadoop/hive/ql/exec/mr/MapredLocalTask.java | 24 +- .../bootstrap/load/table/LoadPartitions.java | 4 +- .../repl/bootstrap/load/table/LoadTable.java | 4 +- .../hadoop/hive/ql/exec/tez/DagUtils.java | 40 +- .../hadoop/hive/ql/exec/tez/TezProcessor.java | 67 +- .../org/apache/hadoop/hive/ql/hooks/Entity.java | 17 +- .../hadoop/hive/ql/hooks/WriteEntity.java | 5 +- .../hive/ql/io/orc/encoded/CacheChunk.java | 12 +- .../ql/io/orc/encoded/EncodedReaderImpl.java | 69 +- .../ql/io/rcfile/stats/PartialScanMapper.java | 180 - .../ql/io/rcfile/stats/PartialScanTask.java | 381 -- .../ql/io/rcfile/stats/PartialScanWork.java | 119 - .../apache/hadoop/hive/ql/metadata/Hive.java | 81 +- .../apache/hadoop/hive/ql/metadata/Table.java | 1 + .../formatting/JsonMetaDataFormatter.java | 5 +- .../formatting/MetaDataFormatUtils.java | 304 +- .../metadata/formatting/MetaDataFormatter.java | 2 +- .../formatting/MetaDataPrettyFormatUtils.java | 244 -- .../formatting/TextMetaDataFormatter.java | 42 +- .../metadata/formatting/TextMetaDataTable.java | 61 + .../hive/ql/optimizer/GenMRTableScan1.java | 57 - .../hive/ql/optimizer/GenMapRedUtils.java | 31 - .../ql/optimizer/QueryPlanPostProcessor.java | 1 - .../hive/ql/optimizer/StatsOptimizer.java | 5 + .../calcite/translator/RexNodeConverter.java | 23 +- .../ql/parse/ColumnStatsSemanticAnalyzer.java | 2 - .../hive/ql/parse/DDLSemanticAnalyzer.java | 25 +- .../hive/ql/parse/FunctionSemanticAnalyzer.java | 9 +- .../org/apache/hadoop/hive/ql/parse/HiveLexer.g | 2 - .../apache/hadoop/hive/ql/parse/HiveParser.g | 6 +- .../hadoop/hive/ql/parse/IdentifiersParser.g | 2 +- .../hive/ql/parse/ImportSemanticAnalyzer.java | 8 +- .../hive/ql/parse/LoadSemanticAnalyzer.java | 4 +- .../hive/ql/parse/ProcessAnalyzeTable.java | 49 - .../hadoop/hive/ql/parse/QBParseInfo.java | 16 - .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 173 +- .../hadoop/hive/ql/parse/repl/CopyUtils.java | 6 +- .../hadoop/hive/ql/parse/repl/dump/Utils.java | 6 + .../parse/spark/SparkProcessAnalyzeTable.java | 51 +- .../hadoop/hive/ql/plan/DescTableDesc.java | 16 +- .../hadoop/hive/ql/plan/LoadTableDesc.java | 44 +- .../apache/hadoop/hive/ql/plan/StatsWork.java | 16 - .../plugin/HiveAuthorizerImpl.java | 1 - .../plugin/HivePrivilegeObject.java | 25 +- .../hadoop/hive/ql/session/SessionState.java | 79 +- .../hive/ql/txn/AcidOpenTxnsCounterService.java | 75 - .../hadoop/hive/ql/util/ResourceDownloader.java | 11 +- .../metastore/txn/TestCompactionTxnHandler.java | 4 +- .../hive/metastore/txn/TestTxnHandler.java | 4 +- .../txn/TestTxnHandlerNoConnectionPool.java | 6 +- .../apache/hadoop/hive/ql/TestTxnCommands.java | 23 +- .../apache/hadoop/hive/ql/TestTxnCommands2.java | 16 +- .../hadoop/hive/ql/TxnCommandsBaseForTests.java | 5 +- .../hadoop/hive/ql/exec/TestExecDriver.java | 4 +- .../hive/ql/lockmgr/TestDbTxnManager.java | 28 +- .../hive/ql/lockmgr/TestDbTxnManager2.java | 356 +- .../hive/ql/metadata/TestHiveCopyFiles.java | 12 +- .../TestSQLStdHiveAccessControllerHS2.java | 3 +- .../hadoop/hive/ql/session/TestAddResource.java | 18 +- .../hive/ql/txn/compactor/CompactorTest.java | 2 +- .../stats_partialscan_autogether.q | 32 - .../stats_partialscan_non_external.q | 5 - .../stats_partialscan_non_native.q | 6 - .../clientnegative/stats_partscan_norcfile.q | 12 - .../queries/clientpositive/describe_pretty.q | 80 - .../queries/clientpositive/escape_comments.q | 1 - .../test/queries/clientpositive/orc_analyze.q | 14 - .../clientpositive/spark_local_queries.q | 16 + .../queries/clientpositive/stats_noscan_2.q | 3 + .../queries/clientpositive/stats_partscan_1.q | 37 - .../clientpositive/stats_partscan_1_23.q | 38 - .../clientnegative/alter_file_format.q.out | 1 - .../alter_partition_coltype_2columns.q.out | 1 - ...alter_partition_coltype_invalidcolname.q.out | 1 - .../alter_partition_coltype_invalidtype.q.out | 1 - ...ter_partition_partial_spec_dyndisabled.q.out | 1 - .../alter_view_as_select_with_partition.q.out | 2 - .../results/clientnegative/desc_failure2.q.out | 1 - .../stats_partialscan_autogether.q.out | 77 - .../stats_partialscan_non_external.q.out | 9 - .../stats_partialscan_non_native.q.out | 11 - .../stats_partscan_norcfile.q.out | 36 - .../clientnegative/wrong_column_type.q.out | 2 +- .../clientpositive/acid_table_stats.q.out | 16 - ql/src/test/results/clientpositive/alter2.q.out | 12 - ql/src/test/results/clientpositive/alter3.q.out | 6 - ql/src/test/results/clientpositive/alter5.q.out | 4 - .../clientpositive/alterColumnStats.q.out | 33 +- .../clientpositive/alterColumnStatsPart.q.out | 11 - .../clientpositive/alter_file_format.q.out | 19 - .../results/clientpositive/alter_merge_2.q.out | 1 - .../clientpositive/alter_merge_2_orc.q.out | 1 - .../clientpositive/alter_merge_stats.q.out | 3 - ...lter_numbuckets_partitioned_table2_h23.q.out | 36 - ...alter_numbuckets_partitioned_table_h23.q.out | 24 - .../alter_partition_change_col.q.out | 13 - .../alter_partition_clusterby_sortby.q.out | 8 - .../alter_partition_coltype.q.out | 4 - .../alter_partition_format_loc.q.out | 6 - .../alter_partition_onto_nocurrent_db.q.out | 2 - .../alter_partition_update_status.q.out | 75 +- .../clientpositive/alter_skewed_table.q.out | 6 - .../alter_table_add_partition.q.out | 10 - .../clientpositive/alter_table_cascade.q.out | 46 - .../alter_table_column_stats.q.out | 1324 +++++-- .../clientpositive/alter_table_not_sorted.q.out | 2 - .../clientpositive/alter_table_serde.q.out | 5 - .../clientpositive/alter_table_serde2.q.out | 8 - .../alter_table_stats_status.q.out | 13 - .../alter_table_update_status.q.out | 720 +++- ..._table_update_status_disable_bitvector.q.out | 720 +++- .../clientpositive/alter_view_as_select.q.out | 3 - .../analyze_table_null_partition.q.out | 1 - .../clientpositive/analyze_tbl_date.q.out | 1 - .../clientpositive/analyze_tbl_part.q.out | 45 +- .../clientpositive/annotate_stats_select.q.out | 16 +- .../clientpositive/authorization_index.q.out | 1 - .../clientpositive/autoColumnStats_3.q.out | 14 - .../clientpositive/autoColumnStats_4.q.out | 3 - .../clientpositive/autoColumnStats_5.q.out | 76 +- .../clientpositive/autoColumnStats_8.q.out | 1 - .../clientpositive/autoColumnStats_9.q.out | 31 +- .../results/clientpositive/auto_join25.q.out | 4 + .../auto_join_without_localtask.q.out | 2 + .../clientpositive/avro_add_column3.q.out | 1 - .../results/clientpositive/avro_decimal.q.out | 15 +- .../clientpositive/avro_decimal_native.q.out | 15 +- .../avro_schema_evolution_native.q.out | 4 - .../beeline/colstats_all_nulls.q.out | 91 + .../beeline/escape_comments.q.out | 32 +- .../test/results/clientpositive/bitvector.q.out | 15 +- .../bucketsortoptimize_insert_8.q.out | 2 + .../clientpositive/colstats_all_nulls.q.out | 30 +- .../columnStatsUpdateForStatsOptimizer_2.q.out | 4 - ...names_with_leading_and_trailing_spaces.q.out | 31 +- .../column_pruner_multiple_children.q.out | 31 +- .../clientpositive/columnstats_infinity.q.out | 3 - .../clientpositive/columnstats_partlvl.q.out | 90 +- .../clientpositive/columnstats_partlvl_dp.q.out | 150 +- .../clientpositive/columnstats_tbllvl.q.out | 105 +- .../test/results/clientpositive/combine3.q.out | 2 - .../results/clientpositive/compustat_avro.q.out | 30 +- .../clientpositive/compute_stats_date.q.out | 30 +- .../confirm_initial_tbl_stats.q.out | 166 +- .../clientpositive/convert_enum_to_string.q.out | 1 - .../create_alter_list_bucketing_table1.q.out | 7 - .../results/clientpositive/create_like.q.out | 10 - .../results/clientpositive/create_like2.q.out | 1 - .../clientpositive/create_like_tbl_props.q.out | 5 - .../clientpositive/create_like_view.q.out | 5 - .../clientpositive/create_or_replace_view.q.out | 9 - .../clientpositive/create_skewed_table1.q.out | 3 - .../create_table_like_stats.q.out | 6 - .../results/clientpositive/create_view.q.out | 14 - .../create_view_defaultformats.q.out | 2 - .../create_view_partitioned.q.out | 7 - .../clientpositive/create_view_translate.q.out | 2 - .../create_with_constraints.q.out | 36 - ql/src/test/results/clientpositive/ctas.q.out | 5 - .../results/clientpositive/ctas_colname.q.out | 7 - .../ctas_uses_database_location.q.out | 1 - .../clientpositive/database_location.q.out | 2 - .../clientpositive/dbtxnmgr_query5.q.out | 1 - .../clientpositive/decimal_precision2.q.out | 8 +- .../results/clientpositive/decimal_serde.q.out | 2 - .../results/clientpositive/decimal_stats.q.out | 15 +- .../clientpositive/default_file_format.q.out | 12 - .../results/clientpositive/deleteAnalyze.q.out | 16 +- .../clientpositive/desc_tbl_part_cols.q.out | 1 - .../describe_comment_indent.q.out | 1 - .../describe_comment_nonascii.q.out | 1 - .../describe_formatted_view_partitioned.q.out | 2 - .../clientpositive/describe_pretty.q.out | 323 -- .../clientpositive/describe_syntax.q.out | 61 +- .../results/clientpositive/describe_table.q.out | 84 +- .../display_colstats_tbllvl.q.out | 180 +- .../clientpositive/drop_partitions_filter.q.out | 1 - .../drop_partitions_filter2.q.out | 1 - .../drop_partitions_filter3.q.out | 1 - .../results/clientpositive/druid_basic1.q.out | 2 - .../results/clientpositive/druid_basic2.q.out | 1 - .../clientpositive/druid_intervals.q.out | 1 - .../clientpositive/druid_timeseries.q.out | 1 - .../results/clientpositive/druid_topn.q.out | 1 - .../encrypted/encryption_insert_values.q.out | 1 - .../encrypted/encryption_move_tbl.q.out | 60 +- .../clientpositive/escape_comments.q.out | 24 - .../clientpositive/exim_02_00_part_empty.q.out | 1 - .../results/clientpositive/exim_02_part.q.out | 1 - .../clientpositive/exim_04_all_part.q.out | 1 - .../clientpositive/exim_04_evolved_parts.q.out | 3 - .../clientpositive/exim_05_some_part.q.out | 1 - .../clientpositive/exim_06_one_part.q.out | 1 - .../exim_07_all_part_over_nonoverlap.q.out | 1 - .../exim_09_part_spec_nonoverlap.q.out | 1 - .../clientpositive/exim_15_external_part.q.out | 1 - .../clientpositive/exim_18_part_external.q.out | 1 - .../exim_19_00_part_external_location.q.out | 1 - .../exim_19_part_external_location.q.out | 1 - .../exim_20_part_managed_location.q.out | 1 - .../clientpositive/exim_hidden_files.q.out | 2 - .../extrapolate_part_stats_full.q.out | 15 +- .../extrapolate_part_stats_partial.q.out | 30 +- .../test/results/clientpositive/fm-sketch.q.out | 75 +- ql/src/test/results/clientpositive/hll.q.out | 75 +- .../clientpositive/index_skewtable.q.out | 1 - .../clientpositive/infer_bucket_sort.q.out | 50 - .../infer_bucket_sort_convert_join.q.out | 5 +- .../infer_bucket_sort_dyn_part.q.out | 16 - .../infer_bucket_sort_grouping_operators.q.out | 12 - .../infer_bucket_sort_list_bucket.q.out | 4 - .../infer_bucket_sort_map_operators.q.out | 8 - .../infer_bucket_sort_merge.q.out | 4 - .../infer_bucket_sort_multi_insert.q.out | 16 - .../infer_bucket_sort_num_buckets.q.out | 4 - .../infer_bucket_sort_reducers_power_two.q.out | 12 - .../test/results/clientpositive/input10.q.out | 1 - .../results/clientpositive/input_part10.q.out | 1 - .../test/results/clientpositive/inputddl2.q.out | 1 - .../test/results/clientpositive/inputddl4.q.out | 2 - .../test/results/clientpositive/inputddl6.q.out | 2 - .../test/results/clientpositive/inputddl7.q.out | 2 - .../test/results/clientpositive/inputddl8.q.out | 1 - .../insert_values_orig_table_use_metadata.q.out | 13 - .../results/clientpositive/lb_fs_stats.q.out | 2 - .../clientpositive/list_bucket_dml_1.q.out | 4 - .../clientpositive/list_bucket_dml_11.q.out | 2 - .../clientpositive/list_bucket_dml_12.q.out | 2 - .../clientpositive/list_bucket_dml_13.q.out | 2 - .../clientpositive/list_bucket_dml_14.q.out | 1 - .../clientpositive/list_bucket_dml_2.q.out | 2 - .../clientpositive/list_bucket_dml_3.q.out | 2 - .../clientpositive/list_bucket_dml_4.q.out | 4 - .../clientpositive/list_bucket_dml_5.q.out | 4 - .../clientpositive/list_bucket_dml_6.q.out | 8 - .../clientpositive/list_bucket_dml_7.q.out | 8 - .../clientpositive/list_bucket_dml_8.q.out | 6 - .../clientpositive/list_bucket_dml_9.q.out | 4 - .../list_bucket_query_multiskew_1.q.out | 2 - .../list_bucket_query_multiskew_2.q.out | 2 - .../list_bucket_query_multiskew_3.q.out | 6 - .../list_bucket_query_oneskew_1.q.out | 2 - .../list_bucket_query_oneskew_2.q.out | 2 - .../list_bucket_query_oneskew_3.q.out | 2 - .../clientpositive/literal_decimal.q.out | 6 +- .../clientpositive/llap/alter_merge_2_orc.q.out | 1 - .../llap/alter_merge_stats_orc.q.out | 8 - .../clientpositive/llap/autoColumnStats_1.q.out | 26 - .../clientpositive/llap/autoColumnStats_2.q.out | 148 +- .../results/clientpositive/llap/bucket5.q.out | 1 - .../clientpositive/llap/bucket_groupby.q.out | 4 - .../columnStatsUpdateForStatsOptimizer_1.q.out | 14 - ...names_with_leading_and_trailing_spaces.q.out | 31 +- .../llap/column_table_stats.q.out | 26 - .../llap/column_table_stats_orc.q.out | 18 - .../llap/columnstats_part_coltype.q.out | 203 +- .../test/results/clientpositive/llap/ctas.q.out | 5 - .../clientpositive/llap/deleteAnalyze.q.out | 16 +- .../llap/dynpart_sort_opt_vectorization.q.out | 32 - .../llap/dynpart_sort_optimization.q.out | 32 - .../llap/dynpart_sort_optimization2.q.out | 24 - .../clientpositive/llap/explainuser_1.q.out | 1 - .../extrapolate_part_stats_partial_ndv.q.out | 330 +- .../llap/infer_bucket_sort_bucketed_table.q.out | 2 - .../llap/list_bucket_dml_10.q.out | 2 - .../clientpositive/llap/load_dyn_part1.q.out | 1 - .../clientpositive/llap/load_dyn_part2.q.out | 1 - .../clientpositive/llap/load_dyn_part3.q.out | 1 - .../clientpositive/llap/load_dyn_part5.q.out | 1 - .../clientpositive/llap/orc_analyze.q.out | 330 -- .../clientpositive/llap/orc_create.q.out | 10 - .../clientpositive/llap/orc_llap_counters.q.out | 1 - .../llap/orc_llap_counters1.q.out | 1 - .../clientpositive/llap/schema_evol_stats.q.out | 12 - .../llap/selectDistinctStar.q.out | 2 - .../results/clientpositive/llap/stats11.q.out | 8 - .../clientpositive/llap/stats_noscan_1.q.out | 18 - .../clientpositive/llap/stats_only_null.q.out | 19 +- .../results/clientpositive/llap/sysdb.q.out | 45 +- .../llap/tez_bmj_schema_evolution.q.out | 1 - .../clientpositive/llap/tez_fsstat.q.out | 2 - .../llap/tez_schema_evolution.q.out | 1 - .../clientpositive/llap/union_fast_stats.q.out | 12 +- .../clientpositive/llap/union_stats.q.out | 5 - .../llap/vector_decimal_math_funcs.q.out | 6 +- .../llap/vectorized_math_funcs.q.out | 2 +- .../results/clientpositive/load_dyn_part1.q.out | 1 - .../clientpositive/load_dyn_part10.q.out | 1 - .../clientpositive/load_dyn_part11.q.out | 1 - .../clientpositive/load_dyn_part12.q.out | 1 - .../clientpositive/load_dyn_part13.q.out | 1 - .../clientpositive/load_dyn_part14.q.out | 1 - .../results/clientpositive/load_dyn_part2.q.out | 1 - .../results/clientpositive/load_dyn_part3.q.out | 1 - .../results/clientpositive/load_dyn_part4.q.out | 1 - .../results/clientpositive/load_dyn_part6.q.out | 1 - .../results/clientpositive/load_dyn_part7.q.out | 1 - .../results/clientpositive/load_dyn_part8.q.out | 1 - .../results/clientpositive/load_dyn_part9.q.out | 1 - .../results/clientpositive/mapjoin_hook.q.out | 3 + .../materialized_view_describe.q.out | 4 - ql/src/test/results/clientpositive/merge3.q.out | 1 - .../clientpositive/named_column_join.q.out | 3 - .../clientpositive/parallel_orderby.q.out | 2 - .../clientpositive/parquet_analyze.q.out | 2 - .../parquet_array_null_element.q.out | 1 - .../results/clientpositive/parquet_create.q.out | 1 - .../parquet_mixed_partition_formats.q.out | 6 - .../clientpositive/parquet_partitioned.q.out | 2 - .../results/clientpositive/parquet_serde.q.out | 6 - .../clientpositive/part_inherit_tbl_props.q.out | 2 - .../part_inherit_tbl_props_empty.q.out | 2 - .../part_inherit_tbl_props_with_star.q.out | 2 - .../clientpositive/partial_column_stats.q.out | 15 +- .../partition_coltype_literals.q.out | 107 +- .../clientpositive/partition_date2.q.out | 2 - .../clientpositive/partition_schema1.q.out | 3 - .../clientpositive/partition_timestamp2.q.out | 2 - .../clientpositive/perf/spark/query14.q.out | 3237 ---------------- .../clientpositive/perf/spark/query21.q.out | 2 +- .../clientpositive/perf/spark/query75.q.out | 72 +- .../clientpositive/perf/tez/query14.q.out | 2 +- .../clientpositive/perf/tez/query21.q.out | 2 +- .../results/clientpositive/quotedid_stats.q.out | 2 - .../clientpositive/rcfile_default_format.q.out | 8 - .../rename_external_partition_location.q.out | 68 +- .../rename_table_update_column_stats.q.out | 180 +- .../clientpositive/repl_2_exim_basic.q.out | 4 - .../clientpositive/repl_3_exim_metadata.q.out | 2 - .../sample_islocalmode_hook_use_metadata.q.out | 1 - .../clientpositive/serde_reported_schema.q.out | 2 - .../clientpositive/show_indexes_syntax.q.out | 2 - .../spark/alter_merge_stats_orc.q.out | 8 - .../spark/avro_decimal_native.q.out | 15 +- .../results/clientpositive/spark/bucket5.q.out | 1 - .../results/clientpositive/spark/ctas.q.out | 5 - .../infer_bucket_sort_bucketed_table.q.out | 2 - .../spark/infer_bucket_sort_map_operators.q.out | 8 - .../spark/infer_bucket_sort_merge.q.out | 4 - .../spark/infer_bucket_sort_num_buckets.q.out | 4 - .../infer_bucket_sort_reducers_power_two.q.out | 12 - .../spark/list_bucket_dml_10.q.out | 2 - .../spark/list_bucket_dml_2.q.out | 2 - .../clientpositive/spark/load_dyn_part1.q.out | 1 - .../clientpositive/spark/load_dyn_part10.q.out | 1 - .../clientpositive/spark/load_dyn_part11.q.out | 1 - .../clientpositive/spark/load_dyn_part12.q.out | 1 - .../clientpositive/spark/load_dyn_part13.q.out | 1 - .../clientpositive/spark/load_dyn_part14.q.out | 1 - .../clientpositive/spark/load_dyn_part2.q.out | 1 - .../clientpositive/spark/load_dyn_part3.q.out | 1 - .../clientpositive/spark/load_dyn_part4.q.out | 1 - .../clientpositive/spark/load_dyn_part5.q.out | 1 - .../clientpositive/spark/load_dyn_part6.q.out | 1 - .../clientpositive/spark/load_dyn_part7.q.out | 1 - .../clientpositive/spark/load_dyn_part8.q.out | 1 - .../clientpositive/spark/load_dyn_part9.q.out | 1 - .../clientpositive/spark/parallel_orderby.q.out | 2 - .../spark/spark_explainuser_1.q.out | 1 - .../spark/spark_local_queries.q.out | 131 + .../results/clientpositive/spark/stats0.q.out | 4 - .../results/clientpositive/spark/stats1.q.out | 2 - .../results/clientpositive/spark/stats10.q.out | 6 - .../results/clientpositive/spark/stats12.q.out | 10 - .../results/clientpositive/spark/stats13.q.out | 12 - .../results/clientpositive/spark/stats14.q.out | 9 - .../results/clientpositive/spark/stats15.q.out | 9 - .../results/clientpositive/spark/stats16.q.out | 2 - .../results/clientpositive/spark/stats18.q.out | 4 - .../results/clientpositive/spark/stats2.q.out | 4 - .../results/clientpositive/spark/stats3.q.out | 3 - .../results/clientpositive/spark/stats5.q.out | 1 - .../results/clientpositive/spark/stats6.q.out | 10 - .../results/clientpositive/spark/stats7.q.out | 6 - .../results/clientpositive/spark/stats8.q.out | 20 - .../results/clientpositive/spark/stats9.q.out | 1 - .../clientpositive/spark/stats_noscan_1.q.out | 18 - .../clientpositive/spark/stats_noscan_2.q.out | 83 +- .../clientpositive/spark/stats_only_null.q.out | 19 +- .../spark/stats_partscan_1_23.q.out | 187 - .../results/clientpositive/spark/statsfs.q.out | 14 - .../clientpositive/spark/union_remove_1.q.out | 1 - .../clientpositive/spark/union_remove_10.q.out | 1 - .../clientpositive/spark/union_remove_11.q.out | 1 - .../clientpositive/spark/union_remove_12.q.out | 1 - .../clientpositive/spark/union_remove_13.q.out | 1 - .../clientpositive/spark/union_remove_14.q.out | 1 - .../clientpositive/spark/union_remove_15.q.out | 2 - .../clientpositive/spark/union_remove_16.q.out | 2 - .../clientpositive/spark/union_remove_17.q.out | 2 - .../clientpositive/spark/union_remove_18.q.out | 2 - .../clientpositive/spark/union_remove_19.q.out | 1 - .../clientpositive/spark/union_remove_2.q.out | 1 - .../clientpositive/spark/union_remove_20.q.out | 1 - .../clientpositive/spark/union_remove_21.q.out | 1 - .../clientpositive/spark/union_remove_22.q.out | 1 - .../clientpositive/spark/union_remove_23.q.out | 1 - .../clientpositive/spark/union_remove_24.q.out | 1 - .../clientpositive/spark/union_remove_25.q.out | 6 - .../clientpositive/spark/union_remove_3.q.out | 1 - .../clientpositive/spark/union_remove_4.q.out | 1 - .../clientpositive/spark/union_remove_5.q.out | 1 - .../clientpositive/spark/union_remove_7.q.out | 1 - .../clientpositive/spark/union_remove_8.q.out | 1 - .../clientpositive/spark/union_remove_9.q.out | 1 - .../spark/vectorized_math_funcs.q.out | 2 +- ql/src/test/results/clientpositive/stats0.q.out | 4 - ql/src/test/results/clientpositive/stats1.q.out | 2 - .../test/results/clientpositive/stats10.q.out | 6 - .../test/results/clientpositive/stats12.q.out | 10 - .../test/results/clientpositive/stats13.q.out | 12 - .../test/results/clientpositive/stats14.q.out | 9 - .../test/results/clientpositive/stats15.q.out | 9 - .../test/results/clientpositive/stats16.q.out | 2 - .../test/results/clientpositive/stats18.q.out | 4 - ql/src/test/results/clientpositive/stats2.q.out | 4 - ql/src/test/results/clientpositive/stats3.q.out | 3 - ql/src/test/results/clientpositive/stats4.q.out | 12 - ql/src/test/results/clientpositive/stats5.q.out | 1 - ql/src/test/results/clientpositive/stats6.q.out | 10 - ql/src/test/results/clientpositive/stats7.q.out | 6 - ql/src/test/results/clientpositive/stats8.q.out | 20 - ql/src/test/results/clientpositive/stats9.q.out | 1 - .../clientpositive/stats_empty_partition.q.out | 2 - .../clientpositive/stats_invalidation.q.out | 2 - .../clientpositive/stats_list_bucket.q.out | 3 - .../results/clientpositive/stats_noscan_1.q.out | 18 - .../results/clientpositive/stats_noscan_2.q.out | 77 +- .../clientpositive/stats_only_null.q.out | 19 +- .../clientpositive/stats_partscan_1.q.out | 215 - .../clientpositive/stats_partscan_1_23.q.out | 190 - .../test/results/clientpositive/statsfs.q.out | 14 - .../temp_table_display_colstats_tbllvl.q.out | 180 +- .../clientpositive/tez/explainanalyze_5.q.out | 1 - .../clientpositive/truncate_column.q.out | 11 - .../results/clientpositive/tunable_ndv.q.out | 120 +- .../clientpositive/unicode_comments.q.out | 5 - .../clientpositive/unicode_notation.q.out | 3 - .../clientpositive/union_pos_alias.q.out | 1 - .../results/clientpositive/union_remove_1.q.out | 1 - .../clientpositive/union_remove_10.q.out | 1 - .../clientpositive/union_remove_11.q.out | 1 - .../clientpositive/union_remove_12.q.out | 1 - .../clientpositive/union_remove_13.q.out | 1 - .../clientpositive/union_remove_14.q.out | 1 - .../clientpositive/union_remove_15.q.out | 2 - .../clientpositive/union_remove_16.q.out | 2 - .../clientpositive/union_remove_17.q.out | 2 - .../clientpositive/union_remove_18.q.out | 2 - .../clientpositive/union_remove_19.q.out | 1 - .../results/clientpositive/union_remove_2.q.out | 1 - .../clientpositive/union_remove_20.q.out | 1 - .../clientpositive/union_remove_21.q.out | 1 - .../clientpositive/union_remove_22.q.out | 1 - .../clientpositive/union_remove_23.q.out | 1 - .../clientpositive/union_remove_24.q.out | 1 - .../clientpositive/union_remove_25.q.out | 6 - .../results/clientpositive/union_remove_3.q.out | 1 - .../results/clientpositive/union_remove_4.q.out | 1 - .../results/clientpositive/union_remove_5.q.out | 1 - .../results/clientpositive/union_remove_7.q.out | 1 - .../results/clientpositive/union_remove_8.q.out | 1 - .../results/clientpositive/union_remove_9.q.out | 1 - .../results/clientpositive/union_stats.q.out | 5 - .../clientpositive/updateAccessTime.q.out | 6 - .../vector_decimal_math_funcs.q.out | 6 +- .../clientpositive/vectorized_math_funcs.q.out | 2 +- .../results/clientpositive/view_alias.q.out | 6 - .../test/results/clientpositive/view_cbo.q.out | 10 - .../apache/hive/service/server/HiveServer2.java | 8 + .../hive-webapps/hiveserver2/hiveserver2.jsp | 16 +- .../hive/spark/client/SparkClientImpl.java | 63 +- standalone-metastore/pom.xml | 20 + .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 2132 +++++----- .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 860 +++- .../gen/thrift/gen-cpp/hive_metastore_types.h | 288 ++ .../hadoop/hive/metastore/api/WMMapping.java | 804 ++++ .../hadoop/hive/metastore/api/WMPool.java | 908 +++++ .../hive/metastore/api/WMResourcePlan.java | 602 +++ .../hadoop/hive/metastore/api/WMTrigger.java | 706 ++++ .../src/gen/thrift/gen-php/metastore/Types.php | 622 +++ .../gen/thrift/gen-py/hive_metastore/ttypes.py | 458 +++ .../gen/thrift/gen-rb/hive_metastore_types.rb | 100 + .../hive/metastore/RunnableConfigurable.java | 26 + .../hive/metastore/conf/MetastoreConf.java | 63 +- .../datasource/BoneCPDataSourceProvider.java | 94 + .../datasource/DataSourceProvider.java | 79 + .../datasource/DataSourceProviderFactory.java | 47 + .../datasource/HikariCPDataSourceProvider.java | 96 + .../hive/metastore/datasource/package-info.java | 23 + .../hadoop/hive/metastore/model/MWMMapping.java | 82 + .../hadoop/hive/metastore/model/MWMPool.java | 89 + .../hive/metastore/model/MWMResourcePlan.java | 95 + .../hadoop/hive/metastore/model/MWMTrigger.java | 78 + .../metastore/security/DelegationTokenTool.java | 251 ++ .../hive/metastore/tools/SQLGenerator.java | 172 + .../txn/AcidOpenTxnsCounterService.java | 63 + .../hive/metastore/txn/CompactionInfo.java | 151 + .../metastore/txn/CompactionTxnHandler.java | 961 +++++ .../hadoop/hive/metastore/txn/TxnDbUtil.java | 381 ++ .../hadoop/hive/metastore/txn/TxnHandler.java | 3643 +++++++++++++++++ .../hadoop/hive/metastore/txn/TxnStore.java | 420 ++ .../hadoop/hive/metastore/txn/TxnUtils.java | 243 ++ .../hadoop/hive/metastore/utils/JavaUtils.java | 46 + .../hive/metastore/utils/StringableMap.java | 80 + .../src/main/resources/package.jdo | 112 + .../src/main/thrift/hive_metastore.thrift | 30 + .../hive/metastore/conf/TestMetastoreConf.java | 17 + .../TestDataSourceProviderFactory.java | 170 + .../metastore/txn/TestTxnHandlerNegative.java | 55 + .../hadoop/hive/metastore/txn/TestTxnUtils.java | 216 ++ storage-api/pom.xml | 7 + .../hive/common/ValidCompactorTxnList.java | 89 + .../hadoop/hive/common/ValidReadTxnList.java | 235 ++ .../apache/hadoop/hive/common/ValidTxnList.java | 112 + .../hive/common/TestValidCompactorTxnList.java | 134 + .../hive/common/TestValidReadTxnList.java | 109 + 627 files changed, 23127 insertions(+), 19526 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java ---------------------------------------------------------------------- diff --cc itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java index e519785,8a6fe3d..2f0efce --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java @@@ -104,7 -105,7 +105,8 @@@ public class TestHiveHistory extends Te db.dropTable(Warehouse.DEFAULT_DATABASE_NAME, src, true, true); db.createTable(src, cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); - db.loadTable(hadoopDataFile[i], src, false, false, false, false, false, null, 0, false); - db.loadTable(hadoopDataFile[i], src, LoadFileType.KEEP_EXISTING, false, false, false, false); ++ db.loadTable(hadoopDataFile[i], src, ++ LoadFileType.KEEP_EXISTING, false, false, false, false, null, 0, false); i++; } http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- diff --cc metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 497b5af,f8b79a0..ff2d988 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@@ -7944,8 -8008,16 +7950,16 @@@ public class HiveMetaStore extends Thri startHouseKeeperService(conf, Class.forName("org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService")); startHouseKeeperService(conf, Class.forName("org.apache.hadoop.hive.ql.txn.AcidCompactionHistoryService")); startHouseKeeperService(conf, Class.forName("org.apache.hadoop.hive.ql.txn.AcidWriteSetService")); + + ThreadPool.initialize(conf); + RunnableConfigurable rc = new AcidOpenTxnsCounterService(); + rc.setConf(conf); + ThreadPool.getPool().scheduleAtFixedRate(rc, 100, MetastoreConf.getTimeVar(conf, + MetastoreConf.ConfVars.COUNT_OPEN_TXNS_INTERVAL, TimeUnit.MILLISECONDS), + TimeUnit.MILLISECONDS); + } - private static void startHouseKeeperService(HiveConf conf, Class c) throws Exception { + private static void startHouseKeeperService(HiveConf conf, Class<?> c) throws Exception { //todo: when metastore adds orderly-shutdown logic, houseKeeper.stop() //should be called form it HouseKeeperService houseKeeper = (HouseKeeperService)c.newInstance(); http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/Driver.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java index 7cc5c30,0dadd51..bef7433 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java @@@ -372,13 -377,10 +373,13 @@@ public class MoveTask extends Task<Move DataContainer dc = null; if (tbd.getPartitionSpec().size() == 0) { dc = new DataContainer(table.getTTable()); + if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { + Utilities.FILE_OP_LOGGER.trace("loadTable called from " + tbd.getSourcePath() + + " into " + tbd.getTable().getTableName()); + } - db.loadTable(tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getReplace(), + db.loadTable(tbd.getSourcePath(), tbd.getTable().getTableName(), tbd.getLoadFileType(), - work.isSrcLocal(), isSkewedStoredAsDirs(tbd), - work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID, - hasFollowingStatsTask()); + work.isSrcLocal(), isSkewedStoredAsDirs(tbd), isFullAcidOp, hasFollowingStatsTask(), + tbd.getTxnId(), tbd.getStmtId(), tbd.isMmTable()); if (work.getOutputs() != null) { DDLTask.addIfAbsentByName(new WriteEntity(table, getWriteType(tbd, work.getLoadTableWork().getWriteType())), work.getOutputs()); @@@ -445,230 -587,6 +446,230 @@@ return (1); } } + + private DataContainer handleStaticParts(Hive db, Table table, LoadTableDesc tbd, + TaskInformation ti) throws HiveException, IOException, InvalidOperationException { + List<String> partVals = MetaStoreUtils.getPvals(table.getPartCols(), tbd.getPartitionSpec()); + db.validatePartitionNameCharacters(partVals); + if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { + Utilities.FILE_OP_LOGGER.trace("loadPartition called from " + tbd.getSourcePath() + + " into " + tbd.getTable().getTableName()); + } + db.loadPartition(tbd.getSourcePath(), tbd.getTable().getTableName(), - tbd.getPartitionSpec(), tbd.getReplace(), ++ tbd.getPartitionSpec(), tbd.getLoadFileType(), + tbd.getInheritTableSpecs(), isSkewedStoredAsDirs(tbd), work.isSrcLocal(), + work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID && + !tbd.isMmTable(), + hasFollowingStatsTask(), tbd.getTxnId(), tbd.getStmtId()); + Partition partn = db.getPartition(table, tbd.getPartitionSpec(), false); + + // See the comment inside updatePartitionBucketSortColumns. + if (!tbd.isMmTable() && (ti.bucketCols != null || ti.sortCols != null)) { + updatePartitionBucketSortColumns(db, table, partn, ti.bucketCols, + ti.numBuckets, ti.sortCols); + } + + DataContainer dc = new DataContainer(table.getTTable(), partn.getTPartition()); + // add this partition to post-execution hook + if (work.getOutputs() != null) { + DDLTask.addIfAbsentByName(new WriteEntity(partn, + getWriteType(tbd, work.getLoadTableWork().getWriteType())), work.getOutputs()); + } + return dc; + } + + private DataContainer handleDynParts(Hive db, Table table, LoadTableDesc tbd, + TaskInformation ti, DynamicPartitionCtx dpCtx) throws HiveException, + IOException, InvalidOperationException { + DataContainer dc; + List<LinkedHashMap<String, String>> dps = Utilities.getFullDPSpecs(conf, dpCtx); + + console.printInfo(System.getProperty("line.separator")); + long startTime = System.currentTimeMillis(); + // load the list of DP partitions and return the list of partition specs + // TODO: In a follow-up to HIVE-1361, we should refactor loadDynamicPartitions + // to use Utilities.getFullDPSpecs() to get the list of full partSpecs. + // After that check the number of DPs created to not exceed the limit and + // iterate over it and call loadPartition() here. + // The reason we don't do inside HIVE-1361 is the latter is large and we + // want to isolate any potential issue it may introduce. + Map<Map<String, String>, Partition> dp = + db.loadDynamicPartitions( + tbd.getSourcePath(), + tbd.getTable().getTableName(), + tbd.getPartitionSpec(), - tbd.getReplace(), ++ tbd.getLoadFileType(), + dpCtx.getNumDPCols(), + (tbd.getLbCtx() == null) ? 0 : tbd.getLbCtx().calculateListBucketingLevel(), + work.getLoadTableWork().getWriteType() != AcidUtils.Operation.NOT_ACID && + !tbd.isMmTable(), + work.getLoadTableWork().getTxnId(), tbd.getStmtId(), hasFollowingStatsTask(), + work.getLoadTableWork().getWriteType()); + + // publish DP columns to its subscribers + if (dps != null && dps.size() > 0) { + pushFeed(FeedType.DYNAMIC_PARTITIONS, dp.values()); + } + + String loadTime = "\t Time taken to load dynamic partitions: " + + (System.currentTimeMillis() - startTime)/1000.0 + " seconds"; + console.printInfo(loadTime); + LOG.info(loadTime); + + if (dp.size() == 0 && conf.getBoolVar(HiveConf.ConfVars.HIVE_ERROR_ON_EMPTY_PARTITION)) { + throw new HiveException("This query creates no partitions." + + " To turn off this error, set hive.error.on.empty.partition=false."); + } + + startTime = System.currentTimeMillis(); + // for each partition spec, get the partition + // and put it to WriteEntity for post-exec hook + for(Map.Entry<Map<String, String>, Partition> entry : dp.entrySet()) { + Partition partn = entry.getValue(); + + // See the comment inside updatePartitionBucketSortColumns. + if (!tbd.isMmTable() && (ti.bucketCols != null || ti.sortCols != null)) { + updatePartitionBucketSortColumns( + db, table, partn, ti.bucketCols, ti.numBuckets, ti.sortCols); + } + + WriteEntity enty = new WriteEntity(partn, + getWriteType(tbd, work.getLoadTableWork().getWriteType())); + if (work.getOutputs() != null) { + DDLTask.addIfAbsentByName(enty, work.getOutputs()); + } + // Need to update the queryPlan's output as well so that post-exec hook get executed. + // This is only needed for dynamic partitioning since for SP the the WriteEntity is + // constructed at compile time and the queryPlan already contains that. + // For DP, WriteEntity creation is deferred at this stage so we need to update + // queryPlan here. + if (queryPlan.getOutputs() == null) { + queryPlan.setOutputs(new LinkedHashSet<WriteEntity>()); + } + queryPlan.getOutputs().add(enty); + + // update columnar lineage for each partition + dc = new DataContainer(table.getTTable(), partn.getTPartition()); + + // Don't set lineage on delete as we don't have all the columns + if (work.getLineagState() != null && + work.getLoadTableWork().getWriteType() != AcidUtils.Operation.DELETE && + work.getLoadTableWork().getWriteType() != AcidUtils.Operation.UPDATE) { + work.getLineagState().setLineage(tbd.getSourcePath(), dc, + table.getCols()); + } + LOG.info("\tLoading partition " + entry.getKey()); + } + console.printInfo("\t Time taken for adding to write entity : " + + (System.currentTimeMillis() - startTime)/1000.0 + " seconds"); + dc = null; // reset data container to prevent it being added again. + return dc; + } + + private void inferTaskInformation(TaskInformation ti) { + // Find the first ancestor of this MoveTask which is some form of map reduce task + // (Either standard, local, or a merge) + while (ti.task.getParentTasks() != null && ti.task.getParentTasks().size() == 1) { + ti.task = (Task)ti.task.getParentTasks().get(0); + // If it was a merge task or a local map reduce task, nothing can be inferred + if (ti.task instanceof MergeFileTask || ti.task instanceof MapredLocalTask) { + break; + } + + // If it's a standard map reduce task, check what, if anything, it inferred about + // the directory this move task is moving + if (ti.task instanceof MapRedTask) { + MapredWork work = (MapredWork)ti.task.getWork(); + MapWork mapWork = work.getMapWork(); + ti.bucketCols = mapWork.getBucketedColsByDirectory().get(ti.path); + ti.sortCols = mapWork.getSortedColsByDirectory().get(ti.path); + if (work.getReduceWork() != null) { + ti.numBuckets = work.getReduceWork().getNumReduceTasks(); + } + + if (ti.bucketCols != null || ti.sortCols != null) { + // This must be a final map reduce task (the task containing the file sink + // operator that writes the final output) + assert work.isFinalMapRed(); + } + break; + } + + // If it's a move task, get the path the files were moved from, this is what any + // preceding map reduce task inferred information about, and moving does not invalidate + // those assumptions + // This can happen when a conditional merge is added before the final MoveTask, but the + // condition for merging is not met, see GenMRFileSink1. + if (ti.task instanceof MoveTask) { + MoveTask mt = (MoveTask)ti.task; + if (mt.getWork().getLoadFileWork() != null) { + ti.path = mt.getWork().getLoadFileWork().getSourcePath().toUri().toString(); + } + } + } + } + + private void checkFileFormats(Hive db, LoadTableDesc tbd, Table table) + throws HiveException { + if (work.getCheckFileFormat()) { + // Get all files from the src directory + FileStatus[] dirs; + ArrayList<FileStatus> files; + FileSystem srcFs; // source filesystem + try { + srcFs = tbd.getSourcePath().getFileSystem(conf); + dirs = srcFs.globStatus(tbd.getSourcePath()); + files = new ArrayList<FileStatus>(); + for (int i = 0; (dirs != null && i < dirs.length); i++) { + files.addAll(Arrays.asList(srcFs.listStatus(dirs[i].getPath(), FileUtils.HIDDEN_FILES_PATH_FILTER))); + // We only check one file, so exit the loop when we have at least + // one. + if (files.size() > 0) { + break; + } + } + } catch (IOException e) { + throw new HiveException( + "addFiles: filesystem error in check phase", e); + } + + // handle file format check for table level + if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVECHECKFILEFORMAT)) { + boolean flag = true; + // work.checkFileFormat is set to true only for Load Task, so assumption here is + // dynamic partition context is null + if (tbd.getDPCtx() == null) { + if (tbd.getPartitionSpec() == null || tbd.getPartitionSpec().isEmpty()) { + // Check if the file format of the file matches that of the table. + flag = HiveFileFormatUtils.checkInputFormat( + srcFs, conf, tbd.getTable().getInputFileFormatClass(), files); + } else { + // Check if the file format of the file matches that of the partition + Partition oldPart = db.getPartition(table, tbd.getPartitionSpec(), false); + if (oldPart == null) { + // this means we have just created a table and are specifying partition in the + // load statement (without pre-creating the partition), in which case lets use + // table input format class. inheritTableSpecs defaults to true so when a new + // partition is created later it will automatically inherit input format + // from table object + flag = HiveFileFormatUtils.checkInputFormat( + srcFs, conf, tbd.getTable().getInputFileFormatClass(), files); + } else { + flag = HiveFileFormatUtils.checkInputFormat( + srcFs, conf, oldPart.getInputFormatClass(), files); + } + } + if (!flag) { + throw new HiveException(ErrorMsg.WRONG_FILE_FORMAT); + } + } else { + LOG.warn("Skipping file format check as dpCtx is not null"); + } + } + } + } + + /** * so to make sure we crate WriteEntity with the right WriteType. This is (at this point) only * for consistency since LockManager (which is the only thing that pays attention to WriteType) http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/exec/ReplCopyTask.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsNoJobTask.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java index 1725629,bdf3710..f11f163 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java @@@ -179,10 -180,11 +180,11 @@@ public class StatsTask extends Task<Sta // work.getLoadTableDesc().getReplace() is true means insert overwrite command // work.getLoadFileDesc().getDestinationCreateTable().isEmpty() means CTAS etc. // acidTable will not have accurate stats unless it is set through analyze command. - if (work.getTableSpecs() == null && AcidUtils.isAcidTable(table)) { + if (work.getTableSpecs() == null && AcidUtils.isFullAcidTable(table)) { StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); } else if (work.getTableSpecs() != null - || (work.getLoadTableDesc() != null && work.getLoadTableDesc().getReplace()) + || (work.getLoadTableDesc() != null + && (work.getLoadTableDesc().getLoadFileType() == LoadFileType.REPLACE_ALL)) || (work.getLoadFileDesc() != null && !work.getLoadFileDesc() .getDestinationCreateTable().isEmpty())) { StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.TRUE); @@@ -283,10 -282,11 +285,11 @@@ // org.apache.hadoop.hive.metastore.api.Partition tPart = partn.getTPartition(); Map<String, String> parameters = tPart.getParameters(); - if (work.getTableSpecs() == null && AcidUtils.isAcidTable(table)) { + if (work.getTableSpecs() == null && AcidUtils.isFullAcidTable(table)) { StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.FALSE); } else if (work.getTableSpecs() != null - || (work.getLoadTableDesc() != null && work.getLoadTableDesc().getReplace()) + || (work.getLoadTableDesc() != null + && (work.getLoadTableDesc().getLoadFileType() == LoadFileType.REPLACE_ALL)) || (work.getLoadFileDesc() != null && !work.getLoadFileDesc() .getDestinationCreateTable().isEmpty())) { StatsSetupConst.setBasicStatsState(parameters, StatsSetupConst.TRUE); http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java index f9c1997,e1969bb..aa3f186 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java @@@ -109,10 -107,6 +107,8 @@@ public final class TaskFactory MergeFileTask.class)); taskvec.add(new TaskTuple<DependencyCollectionWork>(DependencyCollectionWork.class, DependencyCollectionTask.class)); + taskvec.add(new TaskTuple<ImportCommitWork>(ImportCommitWork.class, + ImportCommitTask.class)); - taskvec.add(new TaskTuple<PartialScanWork>(PartialScanWork.class, - PartialScanTask.class)); taskvec.add(new TaskTuple<IndexMetadataChangeWork>(IndexMetadataChangeWork.class, IndexMetadataChangeTask.class)); taskvec.add(new TaskTuple<TezWork>(TezWork.class, TezTask.class)); http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index c702a24,c199780..b6b6d6c --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@@ -3208,21 -3108,15 +3224,22 @@@ public final class Utilities LOG.info("Processing alias " + alias); // The alias may not have any path + Collection<Map.Entry<Path, ArrayList<String>>> pathToAliases = + work.getPathToAliases().entrySet(); + if (!skipDummy) { + // ConcurrentModification otherwise if adding dummy. + pathToAliases = new ArrayList<>(pathToAliases); + } boolean isEmptyTable = true; boolean hasLogged = false; - // Note: this copies the list because createDummyFileForEmptyPartition may modify the map. - for (Path file : new LinkedList<Path>(work.getPathToAliases().keySet())) { + Path path = null; + for (Map.Entry<Path, ArrayList<String>> e : pathToAliases) { - if (lDrvStat != null && lDrvStat.driverState == DriverState.INTERRUPT) + if (lDrvStat != null && lDrvStat.driverState == DriverState.INTERRUPT) { throw new IOException("Operation is Canceled."); + } - List<String> aliases = work.getPathToAliases().get(file); + Path file = e.getKey(); + List<String> aliases = e.getValue(); if (aliases.contains(alias)) { if (file != null) { isEmptyTable = false; http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java index 51c4090,821d7df..0360816 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java @@@ -44,9 -44,8 +44,10 @@@ import org.apache.hadoop.hive.ql.plan.A import org.apache.hadoop.hive.ql.plan.DDLWork; import org.apache.hadoop.hive.ql.plan.ImportTableDesc; import org.apache.hadoop.hive.ql.plan.LoadTableDesc; + import org.apache.hadoop.hive.ql.plan.LoadTableDesc.LoadFileType; import org.apache.hadoop.hive.ql.plan.MoveWork; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.mortbay.jetty.servlet.AbstractSessionManager; import org.datanucleus.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@@ -240,8 -239,7 +241,9 @@@ public class LoadPartitions Path tmpPath) { LoadTableDesc loadTableWork = new LoadTableDesc( tmpPath, Utilities.getTableDesc(table), partSpec.getPartSpec(), - event.replicationSpec().isReplace(), SessionState.get().getTxnMgr().getCurrentTxnId() - event.replicationSpec().isReplace() ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING); ++ event.replicationSpec().isReplace() ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING, ++ SessionState.get().getTxnMgr().getCurrentTxnId() + ); loadTableWork.setInheritTableSpecs(false); MoveWork work = new MoveWork(new HashSet<>(), new HashSet<>(), loadTableWork, null, false, context.sessionStateLineageState); http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java index 65a3a59,25a2532..766a9a9 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java @@@ -41,8 -41,8 +41,9 @@@ import org.apache.hadoop.hive.ql.parse. import org.apache.hadoop.hive.ql.parse.repl.ReplLogger; import org.apache.hadoop.hive.ql.plan.ImportTableDesc; import org.apache.hadoop.hive.ql.plan.LoadTableDesc; + import org.apache.hadoop.hive.ql.plan.LoadTableDesc.LoadFileType; import org.apache.hadoop.hive.ql.plan.MoveWork; +import org.apache.hadoop.hive.ql.session.SessionState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@@ -225,9 -225,8 +226,10 @@@ public class LoadTable ReplCopyTask.getLoadCopyTask(replicationSpec, dataPath, tmpPath, context.hiveConf); LoadTableDesc loadTableWork = new LoadTableDesc( - tmpPath, Utilities.getTableDesc(table), new TreeMap<>(), replicationSpec.isReplace(), + tmpPath, Utilities.getTableDesc(table), new TreeMap<>(), - replicationSpec.isReplace() ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING); ++ replicationSpec.isReplace() ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING, + SessionState.get().getTxnMgr().getCurrentTxnId() + ); MoveWork moveWork = new MoveWork(new HashSet<>(), new HashSet<>(), loadTableWork, null, false, context.sessionStateLineageState); http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 37ee6b6,26003f4..68c9482 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@@ -1638,13 -1634,12 +1639,13 @@@ public class Hive * @throws HiveException */ public void loadPartition(Path loadPath, String tableName, - Map<String, String> partSpec, boolean replace, boolean inheritTableSpecs, - Map<String, String> partSpec, LoadFileType loadFileType, - boolean inheritTableSpecs, boolean isSkewedStoreAsSubdir, - boolean isSrcLocal, boolean isAcid, boolean hasFollowingStatsTask) throws HiveException { ++ Map<String, String> partSpec, LoadFileType loadFileType, boolean inheritTableSpecs, + boolean isSkewedStoreAsSubdir, boolean isSrcLocal, boolean isAcid, + boolean hasFollowingStatsTask, Long txnId, int stmtId) + throws HiveException { Table tbl = getTable(tableName); - loadPartition(loadPath, tbl, partSpec, replace, inheritTableSpecs, + loadPartition(loadPath, tbl, partSpec, loadFileType, inheritTableSpecs, - isSkewedStoreAsSubdir, isSrcLocal, isAcid, hasFollowingStatsTask); + isSkewedStoreAsSubdir, isSrcLocal, isAcid, hasFollowingStatsTask, txnId, stmtId); } /** @@@ -1672,12 -1667,12 +1673,12 @@@ * true if there is a following task which updates the stats, so, this method need not update. * @return Partition object being loaded with data */ - public Partition loadPartition(Path loadPath, Table tbl, - Map<String, String> partSpec, LoadFileType loadFileType, - boolean inheritTableSpecs, boolean isSkewedStoreAsSubdir, - boolean isSrcLocal, boolean isAcid, boolean hasFollowingStatsTask) throws HiveException { - + public Partition loadPartition(Path loadPath, Table tbl, Map<String, String> partSpec, - boolean replace, boolean inheritTableSpecs, boolean isSkewedStoreAsSubdir, ++ LoadFileType loadFileType, boolean inheritTableSpecs, boolean isSkewedStoreAsSubdir, + boolean isSrcLocal, boolean isAcid, boolean hasFollowingStatsTask, Long txnId, int stmtId) + throws HiveException { Path tblDataLocationPath = tbl.getDataLocation(); + boolean isMmTableWrite = AcidUtils.isInsertOnlyTable(tbl.getParameters()); try { // Get the partition object if it already exists Partition oldPart = getPartition(tbl, partSpec, false); @@@ -1723,46 -1719,15 +1724,48 @@@ if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary() && (null != oldPart)) { newFiles = Collections.synchronizedList(new ArrayList<Path>()); } - - if ((loadFileType == LoadFileType.REPLACE_ALL) || (oldPart == null && !isAcid)) { - boolean isAutoPurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge")); - replaceFiles(tbl.getPath(), loadPath, newPartPath, oldPartPath, getConf(), - isSrcLocal, isAutoPurge, newFiles); + // Note: this assumes both paths are qualified; which they are, currently. + if (isMmTableWrite && loadPath.equals(newPartPath)) { + // MM insert query, move itself is a no-op. + if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { + Utilities.FILE_OP_LOGGER.trace("not moving " + loadPath + " to " + newPartPath + " (MM)"); + } + assert !isAcid; + if (areEventsForDmlNeeded(tbl, oldPart)) { + newFiles = listFilesCreatedByQuery(loadPath, txnId, stmtId); + } + if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { + Utilities.FILE_OP_LOGGER.trace("maybe deleting stuff from " + oldPartPath + + " (new " + newPartPath + ") for replace"); + } - if (replace && oldPartPath != null) { ++ if ((loadFileType == LoadFileType.REPLACE_ALL) && oldPartPath != null) { + boolean isAutoPurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge")); + deleteOldPathForReplace(newPartPath, oldPartPath, getConf(), isAutoPurge, + new JavaUtils.IdPathFilter(txnId, stmtId, false, true), true, + tbl.isStoredAsSubDirectories() ? tbl.getSkewedColNames().size() : 0); + } } else { - FileSystem fs = tbl.getDataLocation().getFileSystem(conf); - Hive.copyFiles(conf, loadPath, newPartPath, fs, isSrcLocal, isAcid, - (loadFileType == LoadFileType.OVERWRITE_EXISTING), newFiles); + // Either a non-MM query, or a load into MM table from an external source. + PathFilter filter = FileUtils.HIDDEN_FILES_PATH_FILTER; + Path destPath = newPartPath; + if (isMmTableWrite) { + // We will load into MM directory, and delete from the parent if needed. + destPath = new Path(destPath, AcidUtils.deltaSubdir(txnId, txnId, stmtId)); - filter = replace ? new JavaUtils.IdPathFilter(txnId, stmtId, false, true) : filter; ++ filter = (loadFileType == LoadFileType.REPLACE_ALL) ++ ? new JavaUtils.IdPathFilter(txnId, stmtId, false, true) : filter; + } + if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { + Utilities.FILE_OP_LOGGER.trace("moving " + loadPath + " to " + destPath); + } - if (replace || (oldPart == null && !isAcid)) { ++ if ((loadFileType == LoadFileType.REPLACE_ALL) || (oldPart == null && !isAcid)) { + boolean isAutoPurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge")); + replaceFiles(tbl.getPath(), loadPath, destPath, oldPartPath, getConf(), + isSrcLocal, isAutoPurge, newFiles, filter, isMmTableWrite); + } else { + FileSystem fs = tbl.getDataLocation().getFileSystem(conf); - Hive.copyFiles(conf, loadPath, destPath, fs, isSrcLocal, isAcid, newFiles); ++ copyFiles(conf, loadPath, destPath, fs, isSrcLocal, isAcid, ++ (loadFileType == LoadFileType.OVERWRITE_EXISTING), newFiles); + } } perfLogger.PerfLogEnd("MoveTask", "FileMoves"); Partition newTPart = oldPart != null ? oldPart : new Partition(tbl, partSpec, newPartPath); @@@ -2086,8 -1961,8 +2089,8 @@@ private void constructOneLBLocationMap( * @throws HiveException */ public Map<Map<String, String>, Partition> loadDynamicPartitions(final Path loadPath, - final String tableName, final Map<String, String> partSpec, final boolean replace, + final String tableName, final Map<String, String> partSpec, final LoadFileType loadFileType, - final int numDP, final boolean listBucketingEnabled, final boolean isAcid, final long txnId, + final int numDP, final int numLB, final boolean isAcid, final long txnId, final int stmtId, final boolean hasFollowingStatsTask, final AcidUtils.Operation operation) throws HiveException { @@@ -2137,8 -2007,8 +2140,8 @@@ // load the partition Partition newPartition = loadPartition(partPath, tbl, fullPartSpec, - replace, true, numLB > 0, - loadFileType, true, listBucketingEnabled, - false, isAcid, hasFollowingStatsTask); ++ loadFileType, true, numLB > 0, + false, isAcid, hasFollowingStatsTask, txnId, stmtId); partitionsMap.put(fullPartSpec, newPartition); if (inPlaceEligible) { @@@ -2161,8 -2031,8 +2164,8 @@@ + " partPath=" + partPath + ", " + " table=" + tbl.getTableName() + ", " + " partSpec=" + fullPartSpec + ", " - + " replace=" + replace + ", " + + " loadFileType=" + loadFileType.toString() + ", " - + " listBucketingEnabled=" + listBucketingEnabled + ", " + + " listBucketingLevel=" + numLB + ", " + " isAcid=" + isAcid + ", " + " hasFollowingStatsTask=" + hasFollowingStatsTask, t); throw t; @@@ -2228,47 -2099,28 +2232,51 @@@ * if there is any following stats task * @param isAcid true if this is an ACID based write */ - public void loadTable(Path loadPath, String tableName, boolean replace, boolean isSrcLocal, + public void loadTable(Path loadPath, String tableName, LoadFileType loadFileType, boolean isSrcLocal, - boolean isSkewedStoreAsSubdir, boolean isAcid, boolean hasFollowingStatsTask) - throws HiveException { + boolean isSkewedStoreAsSubdir, boolean isAcid, boolean hasFollowingStatsTask, + Long txnId, int stmtId, boolean isMmTable) throws HiveException { + List<Path> newFiles = null; Table tbl = getTable(tableName); HiveConf sessionConf = SessionState.getSessionConf(); if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary()) { newFiles = Collections.synchronizedList(new ArrayList<Path>()); } - if (loadFileType == LoadFileType.REPLACE_ALL) { - Path tableDest = tbl.getPath(); - boolean isAutopurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge")); - replaceFiles(tableDest, loadPath, tableDest, tableDest, sessionConf, isSrcLocal, isAutopurge, newFiles); + // Note: this assumes both paths are qualified; which they are, currently. + if (isMmTable && loadPath.equals(tbl.getPath())) { + Utilities.FILE_OP_LOGGER.debug("not moving " + loadPath + " to " + tbl.getPath()); - if (replace) { ++ if (loadFileType == LoadFileType.REPLACE_ALL) { + Path tableDest = tbl.getPath(); + boolean isAutopurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge")); + deleteOldPathForReplace(tableDest, tableDest, sessionConf, isAutopurge, + new JavaUtils.IdPathFilter(txnId, stmtId, false, true), true, + tbl.isStoredAsSubDirectories() ? tbl.getSkewedColNames().size() : 0); + } + newFiles = listFilesCreatedByQuery(loadPath, txnId, stmtId); } else { - FileSystem fs; - try { - fs = tbl.getDataLocation().getFileSystem(sessionConf); - copyFiles(sessionConf, loadPath, tbl.getPath(), fs, isSrcLocal, isAcid, - (loadFileType == LoadFileType.OVERWRITE_EXISTING), newFiles); - } catch (IOException e) { - throw new HiveException("addFiles: filesystem error in check phase", e); + // Either a non-MM query, or a load into MM table from an external source. + Path tblPath = tbl.getPath(), destPath = tblPath; + PathFilter filter = FileUtils.HIDDEN_FILES_PATH_FILTER; + if (isMmTable) { + // We will load into MM directory, and delete from the parent if needed. + destPath = new Path(destPath, AcidUtils.deltaSubdir(txnId, txnId, stmtId)); - filter = replace ? new JavaUtils.IdPathFilter(txnId, stmtId, false, true) : filter; ++ filter = loadFileType == LoadFileType.REPLACE_ALL ++ ? new JavaUtils.IdPathFilter(txnId, stmtId, false, true) : filter; + } - Utilities.FILE_OP_LOGGER.debug("moving " + loadPath + " to " + tblPath + " (replace = " + replace + ")"); - if (replace) { ++ Utilities.FILE_OP_LOGGER.debug("moving " + loadPath + " to " + tblPath ++ + " (replace = " + loadFileType + ")"); ++ if (loadFileType == LoadFileType.REPLACE_ALL) { + boolean isAutopurge = "true".equalsIgnoreCase(tbl.getProperty("auto.purge")); + replaceFiles(tblPath, loadPath, destPath, tblPath, + sessionConf, isSrcLocal, isAutopurge, newFiles, filter, isMmTable); + } else { + try { + FileSystem fs = tbl.getDataLocation().getFileSystem(sessionConf); - copyFiles(sessionConf, loadPath, destPath, fs, isSrcLocal, isAcid, newFiles); ++ copyFiles(sessionConf, loadPath, destPath, fs, isSrcLocal, isAcid, ++ loadFileType == LoadFileType.OVERWRITE_EXISTING, newFiles); + } catch (IOException e) { + throw new HiveException("addFiles: filesystem error in check phase", e); + } } } if (!this.getConf().getBoolVar(HiveConf.ConfVars.HIVESTATSAUTOGATHER)) { http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index 9afeadb,51b4f36..1318c18 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@@ -361,38 -344,17 +362,39 @@@ public class ImportSemanticAnalyzer ext return tblDesc; } + private static Task<?> loadTable(URI fromURI, Table table, boolean replace, Path tgtPath, - ReplicationSpec replicationSpec, EximUtil.SemanticAnalyzerWrapperContext x) { + ReplicationSpec replicationSpec, EximUtil.SemanticAnalyzerWrapperContext x, + Long txnId, int stmtId, boolean isSourceMm) { Path dataPath = new Path(fromURI.toString(), EximUtil.DATA_PATH_NAME); - Path tmpPath = x.getCtx().getExternalTmpPath(tgtPath); - Task<?> copyTask = ReplCopyTask.getLoadCopyTask(replicationSpec, dataPath, tmpPath, x.getConf()); - LoadTableDesc loadTableWork = new LoadTableDesc(tmpPath, + Path destPath = !AcidUtils.isInsertOnlyTable(table.getParameters()) ? x.getCtx().getExternalTmpPath(tgtPath) + : new Path(tgtPath, AcidUtils.deltaSubdir(txnId, txnId, stmtId)); + if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { + Utilities.FILE_OP_LOGGER.trace("adding import work for table with source location: " + + dataPath + "; table: " + tgtPath + "; copy destination " + destPath + "; mm " + txnId + + " (src " + isSourceMm + ") for " + (table == null ? "a new table" : table.getTableName())); + } + + Task<?> copyTask = null; + if (replicationSpec.isInReplicationScope()) { + if (isSourceMm || isAcid(txnId)) { + // Note: this is replication gap, not MM gap... Repl V2 is not ready yet. + throw new RuntimeException("Replicating MM and ACID tables is not supported"); + } + copyTask = ReplCopyTask.getLoadCopyTask(replicationSpec, dataPath, destPath, x.getConf()); + } else { + CopyWork cw = new CopyWork(dataPath, destPath, false); + cw.setSkipSourceMmDirs(isSourceMm); + copyTask = TaskFactory.get(cw, x.getConf()); + } + + LoadTableDesc loadTableWork = new LoadTableDesc(destPath, - Utilities.getTableDesc(table), new TreeMap<>(), replace, txnId); + Utilities.getTableDesc(table), new TreeMap<>(), - replace ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING); - Task<?> loadTableTask = TaskFactory.get(new MoveWork(x.getInputs(), - x.getOutputs(), loadTableWork, null, false, SessionState.get().getLineageState()), - x.getConf()); ++ replace ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING, txnId); + loadTableWork.setTxnId(txnId); + loadTableWork.setStmtId(stmtId); + MoveWork mv = new MoveWork(x.getInputs(), x.getOutputs(), loadTableWork, null, false, SessionState.get().getLineageState()); + Task<?> loadTableTask = TaskFactory.get(mv, x.getConf()); copyTask.addDependentTask(loadTableTask); x.getTasks().add(copyTask); return loadTableTask; @@@ -458,39 -414,20 +460,41 @@@ + partSpecToString(partSpec.getPartSpec()) + " with source location: " + srcLocation); Path tgtLocation = new Path(partSpec.getLocation()); - Path tmpPath = x.getCtx().getExternalTmpPath(tgtLocation); - Task<?> copyTask = ReplCopyTask.getLoadCopyTask( - replicationSpec, new Path(srcLocation), tmpPath, x.getConf()); + Path destPath = !AcidUtils.isInsertOnlyTable(table.getParameters()) ? x.getCtx().getExternalTmpPath(tgtLocation) + : new Path(tgtLocation, AcidUtils.deltaSubdir(txnId, txnId, stmtId)); + Path moveTaskSrc = !AcidUtils.isInsertOnlyTable(table.getParameters()) ? destPath : tgtLocation; + if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) { + Utilities.FILE_OP_LOGGER.trace("adding import work for partition with source location: " + + srcLocation + "; target: " + tgtLocation + "; copy dest " + destPath + "; mm " + + txnId + " (src " + isSourceMm + ") for " + partSpecToString(partSpec.getPartSpec())); + } + + + Task<?> copyTask = null; + if (replicationSpec.isInReplicationScope()) { + if (isSourceMm || isAcid(txnId)) { + // Note: this is replication gap, not MM gap... Repl V2 is not ready yet. + throw new RuntimeException("Replicating MM and ACID tables is not supported"); + } + copyTask = ReplCopyTask.getLoadCopyTask( + replicationSpec, new Path(srcLocation), destPath, x.getConf()); + } else { + CopyWork cw = new CopyWork(new Path(srcLocation), destPath, false); + cw.setSkipSourceMmDirs(isSourceMm); + copyTask = TaskFactory.get(cw, x.getConf()); + } + Task<?> addPartTask = TaskFactory.get(new DDLWork(x.getInputs(), x.getOutputs(), addPartitionDesc), x.getConf()); - LoadTableDesc loadTableWork = new LoadTableDesc(tmpPath, - Utilities.getTableDesc(table), + LoadTableDesc loadTableWork = new LoadTableDesc(moveTaskSrc, Utilities.getTableDesc(table), - partSpec.getPartSpec(), replicationSpec.isReplace(), txnId); + partSpec.getPartSpec(), - replicationSpec.isReplace() ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING); ++ replicationSpec.isReplace() ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING, ++ txnId); + loadTableWork.setTxnId(txnId); + loadTableWork.setStmtId(stmtId); loadTableWork.setInheritTableSpecs(false); Task<?> loadPartTask = TaskFactory.get(new MoveWork( - x.getInputs(), x.getOutputs(), loadTableWork, null, false, - SessionState.get().getLineageState()), - x.getConf()); + x.getInputs(), x.getOutputs(), loadTableWork, null, false, SessionState.get().getLineageState()), x.getConf()); copyTask.addDependentTask(loadPartTask); addPartTask.addDependentTask(loadPartTask); x.getTasks().add(copyTask); http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java index 51efde1,033235b..5f2a34e --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java @@@ -50,8 -49,8 +50,9 @@@ import org.apache.hadoop.hive.ql.io.Hiv import org.apache.hadoop.hive.ql.metadata.Hive; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.plan.LoadTableDesc; + import org.apache.hadoop.hive.ql.plan.LoadTableDesc.LoadFileType; import org.apache.hadoop.hive.ql.plan.MoveWork; import org.apache.hadoop.hive.ql.plan.StatsWork; import org.apache.hadoop.hive.ql.session.SessionState; @@@ -283,9 -276,8 +284,10 @@@ public class LoadSemanticAnalyzer exten LoadTableDesc loadTableWork; loadTableWork = new LoadTableDesc(new Path(fromURI), - Utilities.getTableDesc(ts.tableHandle), partSpec, isOverWrite, txnId); + Utilities.getTableDesc(ts.tableHandle), partSpec, - isOverWrite ? LoadFileType.REPLACE_ALL : LoadFileType.KEEP_EXISTING); ++ isOverWrite ? LoadFileType.REPLACE_ALL : LoadFileType.KEEP_EXISTING, txnId); + loadTableWork.setTxnId(txnId); + loadTableWork.setStmtId(stmtId); if (preservePartitionSpecs){ // Note : preservePartitionSpecs=true implies inheritTableSpecs=false but // but preservePartitionSpecs=false(default) here is not sufficient enough http://git-wip-us.apache.org/repos/asf/hive/blob/42335b46/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java index 9ac60b0,c497419..9309fbd --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java @@@ -32,11 -30,8 +30,9 @@@ import org.apache.hadoop.hive.conf.Hive import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.exec.Task; import org.apache.hadoop.hive.ql.exec.TaskFactory; +import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.hooks.WriteEntity; import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; - import org.apache.hadoop.hive.ql.io.rcfile.stats.PartialScanWork; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
