Repository: hive Updated Branches: refs/heads/branch-2.1 4569a22ed -> 24f676e0c
HIVE-14400 : Handle concurrent insert with dynamic partition (Wei Zheng, reviewed by Eugene Koifman) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/24f676e0 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/24f676e0 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/24f676e0 Branch: refs/heads/branch-2.1 Commit: 24f676e0c14f237b96eb429a1a09a7184a7e903a Parents: 4569a22 Author: Wei Zheng <[email protected]> Authored: Tue Aug 2 19:52:57 2016 -0700 Committer: Wei Zheng <[email protected]> Committed: Tue Aug 2 19:54:12 2016 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/metadata/Hive.java | 35 ++++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/24f676e0/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index a8af0b6..988b0b1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -1572,14 +1572,24 @@ public class Hive { StatsSetupConst.TRUE); } MetaStoreUtils.populateQuickStats(HiveStatsUtils.getFileStatusRecurse(newPartPath, -1, newPartPath.getFileSystem(conf)), newTPart.getParameters()); - getMSC().add_partition(newTPart.getTPartition()); - } else { - EnvironmentContext environmentContext = null; - if (hasFollowingStatsTask) { - environmentContext = new EnvironmentContext(); - environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE); + try { + getMSC().add_partition(newTPart.getTPartition()); + } catch (AlreadyExistsException aee) { + // With multiple users concurrently issuing insert statements on the same partition has + // a side effect that some queries may not see a partition at the time when they're issued, + // but will realize the partition is actually there when it is trying to add such partition + // to the metastore and thus get AlreadyExistsException, because some earlier query just created it (race condition). + // For example, imagine such a table is created: + // create table T (name char(50)) partitioned by (ds string); + // and the following two queries are launched at the same time, from different sessions: + // insert into table T partition (ds) values ('Bob', 'today'); -- creates the partition 'today' + // insert into table T partition (ds) values ('Joe', 'today'); -- will fail with AlreadyExistsException + // In that case, we want to retry with alterPartition. + LOG.debug("Caught AlreadyExistsException, trying to alter partition instead"); + setStatsPropAndAlterPartition(hasFollowingStatsTask, tbl, newTPart); } - alterPartition(tbl.getDbName(), tbl.getTableName(), new Partition(tbl, newTPart.getTPartition()), environmentContext); + } else { + setStatsPropAndAlterPartition(hasFollowingStatsTask, tbl, newTPart); } return newTPart; } catch (IOException e) { @@ -1597,6 +1607,17 @@ public class Hive { } } + private void setStatsPropAndAlterPartition(boolean hasFollowingStatsTask, Table tbl, + Partition newTPart) throws HiveException, InvalidOperationException { + EnvironmentContext environmentContext = null; + if (hasFollowingStatsTask) { + environmentContext = new EnvironmentContext(); + environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE); + } + alterPartition(tbl.getDbName(), tbl.getTableName(), new Partition(tbl, newTPart.getTPartition()), + environmentContext); + } + /** * Walk through sub-directory tree to construct list bucketing location map. *
