Repository: hive Updated Branches: refs/heads/master e68783c8e -> f68b5dbb5
HIVE-13592 : metastore calls map is not thread safe (Sergey Shelukhin, reviewed by Aihua Xu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f68b5dbb Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f68b5dbb Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f68b5dbb Branch: refs/heads/master Commit: f68b5dbb59a9e837209e64aefe5aa994476c0bdc Parents: e68783c Author: Sergey Shelukhin <[email protected]> Authored: Wed May 4 17:05:20 2016 -0700 Committer: Sergey Shelukhin <[email protected]> Committed: Wed May 4 17:05:39 2016 -0700 ---------------------------------------------------------------------- .../hive/metastore/RetryingMetaStoreClient.java | 17 +++++++++-------- .../org/apache/hadoop/hive/ql/metadata/Hive.java | 3 ++- 2 files changed, 11 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f68b5dbb/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java index f672adf..3c125e0 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java @@ -25,6 +25,7 @@ import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.lang.reflect.UndeclaredThrowableException; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; @@ -55,14 +56,14 @@ public class RetryingMetaStoreClient implements InvocationHandler { private final IMetaStoreClient base; private final int retryLimit; private final long retryDelaySeconds; - private final Map<String, Long> metaCallTimeMap; + private final ConcurrentHashMap<String, Long> metaCallTimeMap; private final long connectionLifeTimeInMillis; private long lastConnectionTime; private boolean localMetaStore; protected RetryingMetaStoreClient(HiveConf hiveConf, Class<?>[] constructorArgTypes, - Object[] constructorArgs, Map<String, Long> metaCallTimeMap, + Object[] constructorArgs, ConcurrentHashMap<String, Long> metaCallTimeMap, Class<? extends IMetaStoreClient> msClientClass) throws MetaException { this.retryLimit = hiveConf.getIntVar(HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES); @@ -94,7 +95,7 @@ public class RetryingMetaStoreClient implements InvocationHandler { } public static IMetaStoreClient getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader, - Map<String, Long> metaCallTimeMap, String mscClassName, boolean allowEmbedded) + ConcurrentHashMap<String, Long> metaCallTimeMap, String mscClassName, boolean allowEmbedded) throws MetaException { return getProxy(hiveConf, @@ -119,7 +120,7 @@ public class RetryingMetaStoreClient implements InvocationHandler { * Please use getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader) for external purpose. */ public static IMetaStoreClient getProxy(HiveConf hiveConf, Class<?>[] constructorArgTypes, - Object[] constructorArgs, Map<String, Long> metaCallTimeMap, + Object[] constructorArgs, ConcurrentHashMap<String, Long> metaCallTimeMap, String mscClassName) throws MetaException { @SuppressWarnings("unchecked") @@ -202,11 +203,11 @@ public class RetryingMetaStoreClient implements InvocationHandler { private void addMethodTime(Method method, long timeTaken) { String methodStr = getMethodString(method); - Long curTime = metaCallTimeMap.get(methodStr); - if (curTime != null) { - timeTaken += curTime; + while (true) { + Long curTime = metaCallTimeMap.get(methodStr), newTime = timeTaken; + if (curTime != null && metaCallTimeMap.replace(methodStr, curTime, newTime + curTime)) break; + if (curTime == null && (null == metaCallTimeMap.putIfAbsent(methodStr, newTime))) break; } - metaCallTimeMap.put(methodStr, timeTaken); } /** http://git-wip-us.apache.org/repos/asf/hive/blob/f68b5dbb/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 6862f70..f4a9772 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -48,6 +48,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.ConcurrentHashMap; import com.google.common.collect.ImmutableMap; @@ -162,7 +163,7 @@ public class Hive { private UserGroupInformation owner; // metastore calls timing information - private final Map<String, Long> metaCallTimeMap = new HashMap<String, Long>(); + private final ConcurrentHashMap<String, Long> metaCallTimeMap = new ConcurrentHashMap<>(); private static ThreadLocal<Hive> hiveDB = new ThreadLocal<Hive>() { @Override
