This is an automated email from the ASF dual-hosted git repository.
psomogyi pushed a commit to branch branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-1 by this push:
new a09c63e HBASE-22637 fix flaky test in TestMetaTableMetrics
a09c63e is described below
commit a09c63ef26e30d35b7e9b5f0515c0b441ca53b45
Author: Mate Szalay-Beko <[email protected]>
AuthorDate: Wed Jul 3 13:57:16 2019 +0200
HBASE-22637 fix flaky test in TestMetaTableMetrics
Signed-off-by: Peter Somogyi <[email protected]>
---
.../hbase/coprocessor/TestMetaTableMetrics.java | 253 +++++++++++----------
1 file changed, 138 insertions(+), 115 deletions(-)
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMetaTableMetrics.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMetaTableMetrics.java
index 97568fd..0f35d60 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMetaTableMetrics.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMetaTableMetrics.java
@@ -11,17 +11,18 @@
package org.apache.hadoop.hbase.coprocessor;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.HashSet;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import java.util.Set;
@@ -37,13 +38,16 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.JMXListener;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Waiter.Predicate;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.testclassification.CoprocessorTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.hbase.util.Threads;
+import org.hamcrest.CustomTypeSafeMatcher;
+import org.hamcrest.Matcher;
+import org.hamcrest.core.AllOf;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -63,18 +67,20 @@ public class TestMetaTableMetrics {
private static final byte[] QUALIFIER = Bytes.toBytes("q");
private static final int NUM_ROWS = 5;
private static final String value = "foo";
- private static Configuration conf = null;
+ private static final String METRICS_ATTRIBUTE_NAME_PREFIX = "MetaTable_";
+ private static final List<String> METRICS_ATTRIBUTE_NAME_POSTFIXES =
+ Arrays.asList("_count", "_mean_rate", "_1min_rate", "_5min_rate",
"_15min_rate");
private static int connectorPort = 61120;
- final byte[] cf = Bytes.toBytes("info");
- final byte[] col = Bytes.toBytes("any");
- byte[] tablename;
- final int nthreads = 20;
+ private final byte[] cf = Bytes.toBytes("info");
+ private final byte[] col = Bytes.toBytes("any");
+ private byte[] tablename;
+ private final int nthreads = 20;
@BeforeClass
public static void setupBeforeClass() throws Exception {
- conf = UTIL.getConfiguration();
+ Configuration conf = UTIL.getConfiguration();
// Set system coprocessor so it can be applied to meta regions
UTIL.getConfiguration().set("hbase.coprocessor.region.classes",
MetaTableMetrics.class.getName());
@@ -88,11 +94,9 @@ public class TestMetaTableMetrics {
try {
conf.setInt("regionserver.rmi.registry.port", connectorPort);
UTIL.startMiniCluster(1);
- UTIL.createTable(NAME1, new byte[][]{FAMILY});
- LOG.error("util to string" + UTIL.toString());
break;
} catch (Exception e) {
- LOG.debug("Encountered exception when starting cluster. Trying port "
+ connectorPort, e);
+ LOG.debug("Encountered exception when starting cluster. Trying port
{}", connectorPort, e);
try {
// this is to avoid "IllegalStateException: A mini-cluster is
already running"
UTIL.shutdownMiniCluster();
@@ -108,41 +112,120 @@ public class TestMetaTableMetrics {
UTIL.shutdownMiniCluster();
}
- private void writeData(Table t) throws IOException {
- List<Put> puts = new ArrayList<>(NUM_ROWS);
- for (int i = 0; i < NUM_ROWS; i++) {
- Put p = new Put(Bytes.toBytes(i + 1));
- p.addColumn(FAMILY, QUALIFIER, Bytes.toBytes(value));
- puts.add(p);
+ // Verifies that meta table metrics exist in jmx. In case of one table (one
region) with a single
+ // client: 9 metrics
+ // are generated and for each metrics, there should be 5 JMX attributes
produced. e.g. for one
+ // table, there should
+ // be 5 MetaTable_table_<TableName>_request attributes, such as:
+ // - MetaTable_table_TestExampleMetaTableMetricsOne_request_count
+ // - MetaTable_table_TestExampleMetaTableMetricsOne_request_mean_rate
+ // - MetaTable_table_TestExampleMetaTableMetricsOne_request_1min_rate
+ // - MetaTable_table_TestExampleMetaTableMetricsOne_request_5min_rate
+ // - MetaTable_table_TestExampleMetaTableMetricsOne_request_15min_rate
+ @Test
+ public void testMetaTableMetricsInJmx() throws Exception {
+ UTIL.createTable(NAME1, new byte[][]{FAMILY});
+ writeData(NAME1);
+ readingSingleRowFromTheMetaTable();
+ UTIL.deleteTable(NAME1);
+
+ UTIL.waitFor(30000, 2000, true, new Predicate<IOException>() {
+
+ @Override
+ public boolean evaluate() throws IOException {
+ Map<String, Double> jmxMetrics = readMetaTableJmxMetrics();
+ boolean allMetricsFound = AllOf.allOf(
+ containsPositiveJmxAttributesFor("MetaTable_get_request"),
+ containsPositiveJmxAttributesFor("MetaTable_put_request"),
+ containsPositiveJmxAttributesFor("MetaTable_delete_request"),
+
containsPositiveJmxAttributesFor("MetaTable_region_.+_lossy_request"),
+ containsPositiveJmxAttributesFor("MetaTable_table_" + NAME1 +
"_request"),
+ containsPositiveJmxAttributesFor("MetaTable_client_.+_put_request"),
+ containsPositiveJmxAttributesFor("MetaTable_client_.+_get_request"),
+
containsPositiveJmxAttributesFor("MetaTable_client_.+_delete_request"),
+ containsPositiveJmxAttributesFor("MetaTable_client_.+_lossy_request")
+ ).matches(jmxMetrics);
+
+ if (allMetricsFound) {
+ LOG.info("all the meta table metrics found with positive values:
{}", jmxMetrics);
+ } else {
+ LOG.warn("couldn't find all the meta table metrics with positive
values: {}", jmxMetrics);
+ }
+ return allMetricsFound;
+ }
+ });
+
+ }
+
+ @Test
+ public void testConcurrentAccess() {
+ try {
+ tablename = Bytes.toBytes("hbase:meta");
+ int numRows = 3000;
+ int numRowsInTableBefore = UTIL.countRows(TableName.valueOf(tablename));
+ putData(numRows);
+ Thread.sleep(2000);
+ int numRowsInTableAfter = UTIL.countRows(TableName.valueOf(tablename));
+ assertTrue(numRowsInTableAfter >= numRowsInTableBefore + numRows);
+ getData(numRows);
+ } catch (InterruptedException e) {
+ LOG.info("Caught InterruptedException while testConcurrentAccess: {}",
e.getMessage());
+ fail();
+ } catch (IOException e) {
+ LOG.info("Caught IOException while testConcurrentAccess: {}",
e.getMessage());
+ fail();
}
- t.put(puts);
}
- private Set<String> readJmxMetricsWithRetry() throws IOException {
- final int count = 0;
- for (int i = 0; i < 10; i++) {
- Set<String> metrics = readJmxMetrics();
- if (metrics != null) {
- return metrics;
+ private void writeData(TableName tableName) throws IOException {
+ try (Table t = UTIL.getConnection().getTable(tableName)) {
+ List<Put> puts = new ArrayList<>(NUM_ROWS);
+ for (int i = 0; i < NUM_ROWS; i++) {
+ Put p = new Put(Bytes.toBytes(i + 1));
+ p.addColumn(FAMILY, QUALIFIER, Bytes.toBytes(value));
+ puts.add(p);
}
- LOG.warn("Failed to get jmxmetrics... sleeping, retrying; " + i + " of "
+ count + " times");
- Threads.sleep(1000);
+ t.put(puts);
}
- return null;
+ }
+
+ private void readingSingleRowFromTheMetaTable() throws IOException {
+ TableName metaTableName = TableName.valueOf(Bytes.toBytes("hbase:meta"));
+ try (Table metaTable = UTIL.getConnection().getTable(metaTableName)) {
+ Get get = new Get(Bytes.toBytes(1));
+ metaTable.get(get);
+ }
+ }
+
+ private Matcher<Map<String, Double>> containsPositiveJmxAttributesFor(final
String regexp) {
+ return new CustomTypeSafeMatcher<Map<String, Double>>(
+ "failed to find all the 5 positive JMX attributes for: " + regexp) {
+
+ @Override
+ protected boolean matchesSafely(final Map<String, Double> values) {
+ for (String key : values.keySet()) {
+ for (String metricsNamePostfix : METRICS_ATTRIBUTE_NAME_POSTFIXES) {
+ if (key.matches(regexp + metricsNamePostfix) && values.get(key) >
0) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+ };
}
/**
* Read the attributes from Hadoop->HBase->RegionServer->MetaTableMetrics in
JMX
* @throws IOException when fails to retrieve jmx metrics.
*/
- // this method comes from this class: TestStochasticBalancerJmxMetrics with
minor modifications.
- private Set<String> readJmxMetrics() throws IOException {
+ private Map<String, Double> readMetaTableJmxMetrics() throws IOException {
JMXConnector connector = null;
ObjectName target = null;
MBeanServerConnection mb = null;
try {
connector =
-
JMXConnectorFactory.connect(JMXListener.buildJMXServiceURL(connectorPort,
connectorPort));
+
JMXConnectorFactory.connect(JMXListener.buildJMXServiceURL(connectorPort,
connectorPort));
mb = connector.getMBeanServerConnection();
@SuppressWarnings("JdkObsolete")
@@ -150,26 +233,30 @@ public class TestMetaTableMetrics {
pairs.put("service", "HBase");
pairs.put("name", "RegionServer");
pairs.put("sub",
- "Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor"
- + ".MetaTableMetrics");
+
"Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MetaTableMetrics");
target = new ObjectName("Hadoop", pairs);
MBeanInfo beanInfo = mb.getMBeanInfo(target);
- Set<String> existingAttrs = new HashSet<>();
+ Map<String, Double> existingAttrs = new HashMap<>();
for (MBeanAttributeInfo attrInfo : beanInfo.getAttributes()) {
- existingAttrs.add(attrInfo.getName());
+ Object value = mb.getAttribute(target, attrInfo.getName());
+ if (attrInfo.getName().startsWith(METRICS_ATTRIBUTE_NAME_PREFIX)
+ && value instanceof Number) {
+ existingAttrs.put(attrInfo.getName(),
Double.parseDouble(value.toString()));
+ }
}
+ LOG.info("MBean Found: {}", target);
return existingAttrs;
} catch (Exception e) {
- LOG.warn("Failed to get bean." + target, e);
+ LOG.warn("Failed to get Meta Table Metrics bean (will retry later): {}",
target, e);
if (mb != null) {
Set<ObjectInstance> instances = mb.queryMBeans(null, null);
Iterator<ObjectInstance> iterator = instances.iterator();
- LOG.warn("MBean Found:");
+ LOG.debug("All the MBeans we found:");
while (iterator.hasNext()) {
ObjectInstance instance = iterator.next();
- LOG.warn("Class Name: " + instance.getClassName());
- LOG.warn("Object Name: " + instance.getObjectName());
+ LOG.debug("Class and object name: {} [{}]", instance.getClassName(),
+ instance.getObjectName());
}
}
} finally {
@@ -181,75 +268,11 @@ public class TestMetaTableMetrics {
}
}
}
- return null;
- }
-
- // verifies meta table metrics exist from jmx
- // for one table, there should be 5 MetaTable_table_<TableName> metrics.
- // such as:
- // [Time-limited test] example.TestMetaTableMetrics(204): ==
- // MetaTable_table_TestExampleMetaTableMetricsOne_request_count
- // [Time-limited test] example.TestMetaTableMetrics(204): ==
- // MetaTable_table_TestExampleMetaTableMetricsOne_request_mean_rate
- // [Time-limited test] example.TestMetaTableMetrics(204): ==
- // MetaTable_table_TestExampleMetaTableMetricsOne_request_1min_rate
- // [Time-limited test] example.TestMetaTableMetrics(204): ==
- // MetaTable_table_TestExampleMetaTableMetricsOne_request_5min_rate
- // [Time-limited test] example.TestMetaTableMetrics(204): ==
- // MetaTable_table_TestExampleMetaTableMetricsOne_request_15min_rate
- @Test
- public void test() throws IOException, InterruptedException {
- try (Table t = UTIL.getConnection().getTable(NAME1)) {
- writeData(t);
- // Flush the data
- UTIL.flush(NAME1);
- // Issue a compaction
- UTIL.compact(NAME1, true);
- Thread.sleep(2000);
- }
- Set<String> jmxMetrics = readJmxMetricsWithRetry();
- assertNotNull(jmxMetrics);
-
- long name1TableMetricsCount = 0;
- for(String metric : jmxMetrics) {
- if (metric.contains("MetaTable_table_" + NAME1)){
- name1TableMetricsCount++;
- }
- }
- assertEquals(5L, name1TableMetricsCount);
-
- String putWithClientMetricNameRegex = "MetaTable_client_.+_put_request.*";
- long putWithClientMetricsCount = 0;
- for(String metric : jmxMetrics) {
- if(metric.matches(putWithClientMetricNameRegex)) {
- putWithClientMetricsCount++;
- }
- }
- assertEquals(5L, putWithClientMetricsCount);
- }
-
- @Test(timeout = 30000)
- public void testConcurrentAccess() {
- try {
- tablename = Bytes.toBytes("hbase:meta");
- int numRows = 3000;
- int numRowsInTableBefore = UTIL.countRows(TableName.valueOf(tablename));
- putData(numRows);
- Thread.sleep(2000);
- int numRowsInTableAfter = UTIL.countRows(TableName.valueOf(tablename));
- assertTrue(numRowsInTableAfter >= numRowsInTableBefore + numRows);
- getData(numRows);
- } catch (InterruptedException e) {
- LOG.info("Caught InterruptedException while testConcurrentAccess: " +
e.getMessage());
- fail();
- } catch (IOException e) {
- LOG.info("Caught IOException while testConcurrentAccess: " +
e.getMessage());
- fail();
- }
+ return Collections.emptyMap();
}
- public void putData(int nrows) throws InterruptedException {
- LOG.info(String.format("Putting %d rows in hbase:meta", nrows));
+ private void putData(int nrows) throws InterruptedException {
+ LOG.info("Putting {} rows in hbase:meta", nrows);
Thread[] threads = new Thread[nthreads];
for (int i = 1; i <= nthreads; i++) {
threads[i - 1] = new PutThread(1, nrows);
@@ -257,8 +280,8 @@ public class TestMetaTableMetrics {
startThreadsAndWaitToJoin(threads);
}
- public void getData(int nrows) throws InterruptedException {
- LOG.info(String.format("Getting %d rows from hbase:meta", nrows));
+ private void getData(int nrows) throws InterruptedException {
+ LOG.info("Getting {} rows from hbase:meta", nrows);
Thread[] threads = new Thread[nthreads];
for (int i = 1; i <= nthreads; i++) {
threads[i - 1] = new GetThread(1, nrows);
@@ -275,11 +298,11 @@ public class TestMetaTableMetrics {
}
}
- class PutThread extends Thread {
+ private class PutThread extends Thread {
int start;
int end;
- public PutThread(int start, int end) {
+ PutThread(int start, int end) {
this.start = start;
this.end = end;
}
@@ -293,16 +316,16 @@ public class TestMetaTableMetrics {
table.put(p);
}
} catch (IOException e) {
- LOG.info("Caught IOException while PutThread operation: " +
e.getMessage());
+ LOG.warn("Caught IOException while PutThread operation", e);
}
}
}
- class GetThread extends Thread {
+ private class GetThread extends Thread {
int start;
int end;
- public GetThread(int start, int end) {
+ GetThread(int start, int end) {
this.start = start;
this.end = end;
}
@@ -315,7 +338,7 @@ public class TestMetaTableMetrics {
table.get(get);
}
} catch (IOException e) {
- LOG.info("Caught IOException while GetThread operation: " +
e.getMessage());
+ LOG.warn("Caught IOException while GetThread operation", e);
}
}
}
