HIVE-19416 : merge master into branch (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4743c798 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4743c798 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4743c798 Branch: refs/heads/master-txnstats Commit: 4743c7984d8a547cab135f9fc0fc550ac32dd61f Parents: 42a9f3b 6d532e7 Author: sergey <[email protected]> Authored: Fri Jun 22 11:06:12 2018 -0700 Committer: sergey <[email protected]> Committed: Fri Jun 22 11:06:12 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 2 + .../hcatalog/pig/AbstractHCatStorerTest.java | 2 +- .../pig/TestHCatLoaderComplexSchema.java | 3 + .../hive/hcatalog/pig/TestHCatStorer.java | 4 +- .../listener/DummyRawStoreFailEvent.java | 6 + .../test/resources/testconfiguration.properties | 1 + .../apache/hadoop/hive/ql/exec/MoveTask.java | 3 +- .../apache/hadoop/hive/ql/io/IOConstants.java | 1 + .../ql/io/JsonFileStorageFormatDescriptor.java | 51 ++ .../apache/hadoop/hive/ql/metadata/Hive.java | 104 ++-- .../rules/HiveFilterProjectTransposeRule.java | 8 +- .../hive/ql/optimizer/physical/Vectorizer.java | 64 ++- ...he.hadoop.hive.ql.io.StorageFormatDescriptor | 1 + .../hive/ql/io/TestStorageFormatDescriptor.java | 3 + .../hive/ql/metadata/TestHiveCopyFiles.java | 12 +- .../clientpositive/cbo_ppd_non_deterministic.q | 42 ++ .../test/queries/clientpositive/json_serde1.q | 9 +- .../vector_parquet_nested_two_level_complex.q | 67 +++ .../cbo_ppd_non_deterministic.q.out | 195 +++++++ .../results/clientpositive/json_serde1.q.out | 44 +- ...ector_parquet_nested_two_level_complex.q.out | 540 +++++++++++++++++++ .../results/clientpositive/ppd_udf_col.q.out | 62 ++- .../results/clientpositive/union_offcbo.q.out | 64 ++- .../hadoop/hive/metastore/HiveMetaStore.java | 111 ++-- .../hadoop/hive/metastore/ObjectStore.java | 46 ++ .../apache/hadoop/hive/metastore/RawStore.java | 15 + .../hive/metastore/cache/CachedStore.java | 6 + .../hadoop/hive/metastore/utils/FileUtils.java | 14 + .../DummyRawStoreControlledCommit.java | 6 + .../DummyRawStoreForJdoConnection.java | 6 + .../client/MetaStoreFactoryForTests.java | 1 + .../TestTablesCreateDropAlterTruncate.java | 14 +- 32 files changed, 1337 insertions(+), 170 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4743c798/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/listener/DummyRawStoreFailEvent.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4743c798/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- diff --cc ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 3918e62,eab9f4a..f9fab96 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@@ -78,8 -78,13 +78,9 @@@ import org.apache.hadoop.fs.FileSystem import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; + import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hive.common.FileUtils; -import org.apache.hadoop.hive.common.HiveStatsUtils; -import org.apache.hadoop.hive.common.ObjectPair; -import org.apache.hadoop.hive.common.StatsSetupConst; -import org.apache.hadoop.hive.common.ValidTxnWriteIdList; +import org.apache.hadoop.hive.common.*; import org.apache.hadoop.hive.common.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.hive.common.classification.InterfaceStability.Unstable; import org.apache.hadoop.hive.common.log.InPlaceUpdate; @@@ -1815,15 -1775,14 +1811,15 @@@ public class Hive Partition newTPart = oldPart != null ? oldPart : new Partition(tbl, partSpec, newPartPath); alterPartitionSpecInMemory(tbl, partSpec, newTPart.getTPartition(), inheritTableSpecs, newPartPath.toString()); validatePartition(newTPart); + setTableSnapshotForTransactionalPartition(conf, newTPart); + // If config is set, table is not temporary and partition being inserted exists, capture + // the list of files added. For not yet existing partitions (insert overwrite to new partition + // or dynamic partition inserts), the add partition event will capture the list of files added. // Generate an insert event only if inserting into an existing partition // When inserting into a new partition, the add partition event takes care of insert event - if ((null != oldPart) && (null != newFiles)) { + if (conf.getBoolVar(ConfVars.FIRE_EVENTS_FOR_DML) && !tbl.isTemporary() && (null != oldPart)) { fireInsertEvent(tbl, partSpec, (loadFileType == LoadFileType.REPLACE_ALL), newFiles); - } else { - LOG.debug("No new files were created, and is not a replace, or we're inserting into a " - + "partition that does not exist yet. Skipping generating INSERT event."); } // column stats will be inaccurate http://git-wip-us.apache.org/repos/asf/hive/blob/4743c798/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4743c798/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4743c798/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RawStore.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4743c798/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4743c798/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4743c798/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java ----------------------------------------------------------------------
