HIVE-11903 : Add lock metrics to HS2 (Yongzhi Chen via Szehon)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/50b6d0c6 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/50b6d0c6 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/50b6d0c6 Branch: refs/heads/llap Commit: 50b6d0c6bbf3ba838d3851ccc7cad0575c39732c Parents: 274847e Author: Szehon Ho <[email protected]> Authored: Wed Sep 30 12:21:30 2015 -0700 Committer: Szehon Ho <[email protected]> Committed: Wed Sep 30 12:21:30 2015 -0700 ---------------------------------------------------------------------- .../common/metrics/common/MetricsConstant.java | 5 ++ .../hadoop/hive/ql/lockmgr/DbLockManager.java | 21 ++++++++ .../zookeeper/ZooKeeperHiveLockManager.java | 41 ++++++++++++++++ .../zookeeper/TestZookeeperLockManager.java | 50 ++++++++++++++++++++ 4 files changed, 117 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/50b6d0c6/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java index 13c3cf9..88a3c29 100644 --- a/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java +++ b/common/src/java/org/apache/hadoop/hive/common/metrics/common/MetricsConstant.java @@ -33,4 +33,9 @@ public class MetricsConstant { public static String JDO_ROLLBACK_TRANSACTIONS = "rollbacked_jdo_transactions"; public static String JDO_COMMIT_TRANSACTIONS = "committed_jdo_transactions"; public static String JDO_OPEN_TRANSACTIONS = "opened_jdo_transactions"; + + public static String METASTORE_HIVE_LOCKS = "metastore_hive_locks"; + public static String ZOOKEEPER_HIVE_SHAREDLOCKS = "zookeeper_hive_sharedlocks"; + public static String ZOOKEEPER_HIVE_EXCLUSIVELOCKS = "zookeeper_hive_exclusivelocks"; + public static String ZOOKEEPER_HIVE_SEMISHAREDLOCKS = "zookeeper_hive_semisharedlocks"; } http://git-wip-us.apache.org/repos/asf/hive/blob/50b6d0c6/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java index 82e227f..bb9da9d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java @@ -20,6 +20,9 @@ package org.apache.hadoop.hive.ql.lockmgr; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.common.JavaUtils; +import org.apache.hadoop.hive.common.metrics.common.Metrics; +import org.apache.hadoop.hive.common.metrics.common.MetricsConstant; +import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.*; @@ -99,6 +102,16 @@ public class DbLockManager implements HiveLockManager{ throw new LockException(ErrorMsg.LOCK_CANNOT_BE_ACQUIRED.getMsg()); } acquiredLocks.add(hl); + + Metrics metrics = MetricsFactory.getInstance(); + if (metrics != null) { + try { + metrics.incrementCounter(MetricsConstant.METASTORE_HIVE_LOCKS); + } catch (Exception e) { + LOG.warn("Error Reporting hive client metastore lock operation to Metrics system", e); + } + } + return res.getState(); } catch (NoSuchTxnException e) { LOG.error("Metastore could not find txnid " + lock.getTxnid()); @@ -133,6 +146,14 @@ public class DbLockManager implements HiveLockManager{ LOG.debug("Unlocking " + hiveLock); client.unlock(lockId); boolean removed = locks.remove(hiveLock); + Metrics metrics = MetricsFactory.getInstance(); + if (metrics != null) { + try { + metrics.decrementCounter(MetricsConstant.METASTORE_HIVE_LOCKS); + } catch (Exception e) { + LOG.warn("Error Reporting hive client metastore unlock operation to Metrics system", e); + } + } LOG.debug("Removed a lock " + removed); } catch (NoSuchLockException e) { LOG.error("Metastore could find no record of lock " + JavaUtils.lockIdToString(lockId)); http://git-wip-us.apache.org/repos/asf/hive/blob/50b6d0c6/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java index fb954d8..7c7a8d1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/zookeeper/ZooKeeperHiveLockManager.java @@ -19,8 +19,12 @@ package org.apache.hadoop.hive.ql.lockmgr.zookeeper; import com.google.common.annotations.VisibleForTesting; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.metrics.common.Metrics; +import org.apache.hadoop.hive.common.metrics.common.MetricsConstant; +import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.lockmgr.*; @@ -402,7 +406,25 @@ public class ZooKeeperHiveLockManager implements HiveLockManager { return null; } } + Metrics metrics = MetricsFactory.getInstance(); + if (metrics != null) { + try { + switch(mode) { + case EXCLUSIVE: + metrics.incrementCounter(MetricsConstant.ZOOKEEPER_HIVE_EXCLUSIVELOCKS); + break; + case SEMI_SHARED: + metrics.incrementCounter(MetricsConstant.ZOOKEEPER_HIVE_SEMISHAREDLOCKS); + break; + default: + metrics.incrementCounter(MetricsConstant.ZOOKEEPER_HIVE_SHAREDLOCKS); + break; + } + } catch (Exception e) { + LOG.warn("Error Reporting hive client zookeeper lock operation to Metrics system", e); + } + } return new ZooKeeperHiveLock(res, key, mode); } @@ -438,6 +460,7 @@ public class ZooKeeperHiveLockManager implements HiveLockManager { @VisibleForTesting static void unlockPrimitive(HiveLock hiveLock, String parent, CuratorFramework curatorFramework) throws LockException { ZooKeeperHiveLock zLock = (ZooKeeperHiveLock)hiveLock; + HiveLockMode lMode = hiveLock.getHiveLockMode(); HiveLockObject obj = zLock.getHiveLockObject(); String name = getLastObjectName(parent, obj); try { @@ -448,6 +471,24 @@ public class ZooKeeperHiveLockManager implements HiveLockManager { if (children == null || children.isEmpty()) { curatorFramework.delete().forPath(name); } + Metrics metrics = MetricsFactory.getInstance(); + if (metrics != null) { + try { + switch(lMode) { + case EXCLUSIVE: + metrics.decrementCounter(MetricsConstant.ZOOKEEPER_HIVE_EXCLUSIVELOCKS); + break; + case SEMI_SHARED: + metrics.decrementCounter(MetricsConstant.ZOOKEEPER_HIVE_SEMISHAREDLOCKS); + break; + default: + metrics.decrementCounter(MetricsConstant.ZOOKEEPER_HIVE_SHAREDLOCKS); + break; + } + } catch (Exception e) { + LOG.warn("Error Reporting hive client zookeeper unlock operation to Metrics system", e); + } + } } catch (KeeperException.NoNodeException nne) { //can happen in retrying deleting the zLock after exceptions like InterruptedException //or in a race condition where parent has already been deleted by other process when it http://git-wip-us.apache.org/repos/asf/hive/blob/50b6d0c6/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java index 4a1ef2e..7fcaa22 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java @@ -18,7 +18,14 @@ package org.apache.hadoop.hive.ql.lockmgr.zookeeper; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Paths; + +import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; +import org.apache.hadoop.hive.common.metrics.metrics2.MetricsReporting; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.lockmgr.HiveLockManagerCtx; import org.apache.hadoop.hive.ql.lockmgr.HiveLockMode; import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject; import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData; @@ -33,6 +40,9 @@ import org.junit.Before; import org.junit.After; import org.junit.Test; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + public class TestZookeeperLockManager { private HiveConf conf; @@ -110,5 +120,45 @@ public class TestZookeeperLockManager { conf.setVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT, "9999"); Assert.assertEquals("node1:5666,node2:9999,node3:9999", ZooKeeperHiveHelper.getQuorumServers(conf)); } + + @Test + public void testMetrics() throws Exception{ + conf.setVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_QUORUM, "localhost"); + conf.setVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT, String.valueOf(server.getPort())); + File workDir = new File(System.getProperty("test.tmp.dir")); + File jsonReportFile = new File(workDir, "json_reportingzk1"); + jsonReportFile.delete(); + conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_METRICS_ENABLED, true); + conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + "," + MetricsReporting.JMX.name()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms"); + MetricsFactory.init(conf); + + HiveLockManagerCtx ctx = new HiveLockManagerCtx(conf); + ZooKeeperHiveLockManager zMgr= new ZooKeeperHiveLockManager(); + zMgr.setContext(ctx); + ZooKeeperHiveLock curLock = zMgr.lock(hiveLock, HiveLockMode.SHARED, false); + Thread.sleep(2000); + byte[] jsonData = Files.readAllBytes(Paths.get(jsonReportFile.getAbsolutePath())); + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode rootNode = objectMapper.readTree(jsonData); + JsonNode countersNode = rootNode.path("counters"); + JsonNode zkLockNode = countersNode.path("zookeeper_hive_sharedlocks"); + JsonNode zkLockCountNode = zkLockNode.path("count"); + Assert.assertTrue(zkLockCountNode.asInt() == 1); + + zMgr.unlock(curLock); + Thread.sleep(2000); + jsonData = Files.readAllBytes(Paths.get(jsonReportFile.getAbsolutePath())); + objectMapper = new ObjectMapper(); + rootNode = objectMapper.readTree(jsonData); + countersNode = rootNode.path("counters"); + zkLockNode = countersNode.path("zookeeper_hive_sharedlocks"); + zkLockCountNode = zkLockNode.path("count"); + Assert.assertTrue(zkLockCountNode.asInt() == 0); + zMgr.close(); + } + }
