Author: gates Date: Fri Dec 5 21:05:54 2014 New Revision: 1643433 URL: http://svn.apache.org/r1643433 Log: HIVE-8797 Simultaneous dynamic inserts can result in "partition already exists" error (Alan Gates, reviewied by Thejas Nair)
Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/CheckJDOException.java Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Added: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/CheckJDOException.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/CheckJDOException.java?rev=1643433&view=auto ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/CheckJDOException.java (added) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/CheckJDOException.java Fri Dec 5 21:05:54 2014 @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.metadata; + +import javax.jdo.JDODataStoreException; + +/** + * Check if this is a javax.jdo.JDODataStoreException + */ +public class CheckJDOException { + + public static boolean isJDODataStoreException(Exception e) { + return (e instanceof JDODataStoreException); + } + +} Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1643433&r1=1643432&r2=1643433&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Fri Dec 5 21:05:54 2014 @@ -116,6 +116,7 @@ import org.apache.thrift.TException; import com.google.common.collect.Sets; + /** * This class has functions that implement meta data/DDL operations using calls * to the metastore. @@ -1740,31 +1741,34 @@ private void constructOneLBLocationMap(F if (tpart == null) { LOG.debug("creating partition for table " + tbl.getTableName() + " with partition spec : " + partSpec); - tpart = getMSC().appendPartition(tbl.getDbName(), tbl.getTableName(), pvals); + try { + tpart = getMSC().appendPartition(tbl.getDbName(), tbl.getTableName(), pvals); + } catch (AlreadyExistsException aee) { + LOG.debug("Caught already exists exception, trying to alter partition instead"); + tpart = getMSC().getPartitionWithAuthInfo(tbl.getDbName(), + tbl.getTableName(), pvals, getUserName(), getGroupNames()); + alterPartitionSpec(tbl, partSpec, tpart, inheritTableSpecs, partPath); + } catch (Exception e) { + if (CheckJDOException.isJDODataStoreException(e)) { + // Using utility method above, so that JDODataStoreException doesn't + // have to be used here. This helps avoid adding jdo dependency for + // hcatalog client uses + LOG.debug("Caught JDO exception, trying to alter partition instead"); + tpart = getMSC().getPartitionWithAuthInfo(tbl.getDbName(), + tbl.getTableName(), pvals, getUserName(), getGroupNames()); + if (tpart == null) { + // This means the exception was caused by something other than a race condition + // in creating the partition, since the partition still doesn't exist. + throw e; + } + alterPartitionSpec(tbl, partSpec, tpart, inheritTableSpecs, partPath); + } else { + throw e; + } + } } else { - LOG.debug("altering partition for table " + tbl.getTableName() - + " with partition spec : " + partSpec); - if (inheritTableSpecs) { - tpart.getSd().setOutputFormat(tbl.getTTable().getSd().getOutputFormat()); - tpart.getSd().setInputFormat(tbl.getTTable().getSd().getInputFormat()); - tpart.getSd().getSerdeInfo().setSerializationLib(tbl.getSerializationLib()); - tpart.getSd().getSerdeInfo().setParameters( - tbl.getTTable().getSd().getSerdeInfo().getParameters()); - tpart.getSd().setBucketCols(tbl.getBucketCols()); - tpart.getSd().setNumBuckets(tbl.getNumBuckets()); - tpart.getSd().setSortCols(tbl.getSortCols()); - } - if (partPath == null || partPath.trim().equals("")) { - throw new HiveException("new partition path should not be null or empty."); - } - tpart.getSd().setLocation(partPath); - tpart.getParameters().put(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK,"true"); - String fullName = tbl.getTableName(); - if (!org.apache.commons.lang.StringUtils.isEmpty(tbl.getDbName())) { - fullName = tbl.getDbName() + "." + tbl.getTableName(); - } - alterPartition(fullName, new Partition(tbl, tpart)); + alterPartitionSpec(tbl, partSpec, tpart, inheritTableSpecs, partPath); } } if (tpart == null) { @@ -1777,6 +1781,35 @@ private void constructOneLBLocationMap(F return new Partition(tbl, tpart); } + private void alterPartitionSpec(Table tbl, + Map<String, String> partSpec, + org.apache.hadoop.hive.metastore.api.Partition tpart, + boolean inheritTableSpecs, + String partPath) throws HiveException, InvalidOperationException { + LOG.debug("altering partition for table " + tbl.getTableName() + " with partition spec : " + + partSpec); + if (inheritTableSpecs) { + tpart.getSd().setOutputFormat(tbl.getTTable().getSd().getOutputFormat()); + tpart.getSd().setInputFormat(tbl.getTTable().getSd().getInputFormat()); + tpart.getSd().getSerdeInfo().setSerializationLib(tbl.getSerializationLib()); + tpart.getSd().getSerdeInfo().setParameters( + tbl.getTTable().getSd().getSerdeInfo().getParameters()); + tpart.getSd().setBucketCols(tbl.getBucketCols()); + tpart.getSd().setNumBuckets(tbl.getNumBuckets()); + tpart.getSd().setSortCols(tbl.getSortCols()); + } + if (partPath == null || partPath.trim().equals("")) { + throw new HiveException("new partition path should not be null or empty."); + } + tpart.getSd().setLocation(partPath); + tpart.getParameters().put(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK,"true"); + String fullName = tbl.getTableName(); + if (!org.apache.commons.lang.StringUtils.isEmpty(tbl.getDbName())) { + fullName = tbl.getDbName() + "." + tbl.getTableName(); + } + alterPartition(fullName, new Partition(tbl, tpart)); + } + public boolean dropPartition(String tblName, List<String> part_vals, boolean deleteData) throws HiveException { String[] names = Utilities.getDbTableName(tblName);