This is an automated email from the ASF dual-hosted git repository.
leesf pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 0b52ae3 [HUDI-209] Implement JMX metrics reporter (#1045)
0b52ae3 is described below
commit 0b52ae3ac2685c5afa7821d663854b526b5a1cff
Author: ForwardXu <forwardxu...@gmail.com>
AuthorDate: Thu Nov 28 19:17:34 2019 +0800
[HUDI-209] Implement JMX metrics reporter (#1045)
---
.../apache/hudi/config/HoodieMetricsConfig.java | 8 ++
.../org/apache/hudi/config/HoodieWriteConfig.java | 8 ++
.../apache/hudi/metrics/JmxMetricsReporter.java | 85 ++++++++++++++++++++++
.../hudi/metrics/MetricsReporterFactory.java | 3 +
.../apache/hudi/metrics/MetricsReporterType.java | 2 +-
...oodieMetrics.java => TestHoodieJmxMetrics.java} | 28 ++++---
.../org/apache/hudi/metrics/TestHoodieMetrics.java | 4 +-
pom.xml | 2 +-
8 files changed, 123 insertions(+), 17 deletions(-)
diff --git
a/hudi-client/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java
b/hudi-client/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java
index 8b911e8..4bb8a3f 100644
--- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java
+++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java
@@ -47,6 +47,14 @@ public class HoodieMetricsConfig extends DefaultHoodieConfig
{
public static final String GRAPHITE_SERVER_PORT = GRAPHITE_PREFIX + ".port";
public static final int DEFAULT_GRAPHITE_SERVER_PORT = 4756;
+ // Jmx
+ public static final String JMX_PREFIX = METRIC_PREFIX + ".jmx";
+ public static final String JMX_HOST = JMX_PREFIX + ".host";
+ public static final String DEFAULT_JMX_HOST = "localhost";
+
+ public static final String JMX_PORT = JMX_PREFIX + ".port";
+ public static final int DEFAULT_JMX_PORT = 9889;
+
public static final String GRAPHITE_METRIC_PREFIX = GRAPHITE_PREFIX +
".metric.prefix";
private HoodieMetricsConfig(Properties props) {
diff --git
a/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
b/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index 05c8406..3743116 100644
--- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -474,6 +474,14 @@ public class HoodieWriteConfig extends DefaultHoodieConfig
{
return props.getProperty(HoodieMetricsConfig.GRAPHITE_METRIC_PREFIX);
}
+ public String getJmxHost() {
+ return props.getProperty(HoodieMetricsConfig.JMX_HOST);
+ }
+
+ public int getJmxPort() {
+ return Integer.parseInt(props.getProperty(HoodieMetricsConfig.JMX_PORT));
+ }
+
/**
* memory configs
*/
diff --git
a/hudi-client/src/main/java/org/apache/hudi/metrics/JmxMetricsReporter.java
b/hudi-client/src/main/java/org/apache/hudi/metrics/JmxMetricsReporter.java
new file mode 100644
index 0000000..7bc73d2
--- /dev/null
+++ b/hudi-client/src/main/java/org/apache/hudi/metrics/JmxMetricsReporter.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.metrics;
+
+import com.google.common.base.Preconditions;
+import java.io.Closeable;
+
+import java.lang.management.ManagementFactory;
+import java.rmi.registry.LocateRegistry;
+import javax.management.remote.JMXConnectorServer;
+import javax.management.remote.JMXConnectorServerFactory;
+import javax.management.remote.JMXServiceURL;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+/**
+ * Implementation of Jmx reporter, which used to report jmx metric.
+ */
+public class JmxMetricsReporter extends MetricsReporter {
+
+ private static Logger logger =
LogManager.getLogger(JmxMetricsReporter.class);
+ private final JMXConnectorServer connector;
+ private String host;
+ private int port;
+
+ public JmxMetricsReporter(HoodieWriteConfig config) {
+ try {
+ // Check the host and port here
+ this.host = config.getJmxHost();
+ this.port = config.getJmxPort();
+ if (host == null || port == 0) {
+ throw new RuntimeException(
+ String.format("Jmx cannot be initialized with host[%s] and
port[%s].",
+ host, port));
+ }
+ LocateRegistry.createRegistry(port);
+ String serviceUrl =
+ "service:jmx:rmi://" + host + ":" + port + "/jndi/rmi://" + host +
":" + port + "/jmxrmi";
+ JMXServiceURL url = new JMXServiceURL(serviceUrl);
+ this.connector = JMXConnectorServerFactory
+ .newJMXConnectorServer(url, null,
ManagementFactory.getPlatformMBeanServer());
+ } catch (Exception e) {
+ String msg = "Jmx initialize failed: ";
+ logger.error(msg, e);
+ throw new HoodieException(msg, e);
+ }
+ }
+
+ @Override
+ public void start() {
+ try {
+ Preconditions.checkNotNull(connector, "Cannot start as the jmxReporter
is null.");
+ connector.start();
+ } catch (Exception e) {
+ throw new HoodieException(e);
+ }
+ }
+
+ @Override
+ public void report() {
+ }
+
+ @Override
+ public Closeable getReporter() {
+ return null;
+ }
+}
diff --git
a/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java
b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java
index 195d147..962abd4 100644
---
a/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java
+++
b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java
@@ -41,6 +41,9 @@ public class MetricsReporterFactory {
case INMEMORY:
reporter = new InMemoryMetricsReporter();
break;
+ case JMX:
+ reporter = new JmxMetricsReporter(config);
+ break;
default:
logger.error("Reporter type[" + type + "] is not supported.");
break;
diff --git
a/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterType.java
b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterType.java
index 6d7ccd3..eeec289 100644
--- a/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterType.java
+++ b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterType.java
@@ -22,5 +22,5 @@ package org.apache.hudi.metrics;
* Types of the reporter. Right now we only support Graphite. We can include
JMX and CSV in the future.
*/
public enum MetricsReporterType {
- GRAPHITE, INMEMORY
+ GRAPHITE, INMEMORY, JMX
}
diff --git
a/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
b/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java
similarity index 71%
copy from
hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
copy to
hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java
index e69afb8..7260774 100644
--- a/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
+++
b/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java
@@ -18,31 +18,35 @@
package org.apache.hudi.metrics;
-import org.apache.hudi.config.HoodieWriteConfig;
-
-import org.junit.Before;
-import org.junit.Test;
-
import static org.apache.hudi.metrics.Metrics.registerGauge;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-public class TestHoodieMetrics {
+import org.apache.hudi.config.HoodieMetricsConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import org.junit.Test;
- private HoodieMetrics metrics = null;
+/**
+ * Test for the Jmx metrics report.
+ */
+public class TestHoodieJmxMetrics extends TestHoodieMetrics {
- @Before
+ @Override
public void start() {
HoodieWriteConfig config = mock(HoodieWriteConfig.class);
when(config.isMetricsOn()).thenReturn(true);
-
when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.INMEMORY);
- metrics = new HoodieMetrics(config, "raw_table");
+ when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.JMX);
+ when(config.getJmxHost()).thenReturn(HoodieMetricsConfig.DEFAULT_JMX_HOST);
+ when(config.getJmxPort()).thenReturn(HoodieMetricsConfig.DEFAULT_JMX_PORT);
+ new HoodieMetrics(config, "raw_table");
}
@Test
public void testRegisterGauge() {
- registerGauge("metric1", 123L);
-
assertTrue(Metrics.getInstance().getRegistry().getGauges().get("metric1").getValue().toString().equals("123"));
+ registerGauge("jmx_metric", 123L);
+ assertTrue(Metrics.getInstance().getRegistry().getGauges()
+ .get("jmx_metric").getValue().toString().equals("123"));
}
}
diff --git
a/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
b/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
index e69afb8..058c7b9 100644
--- a/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
+++ b/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
@@ -30,14 +30,12 @@ import static org.mockito.Mockito.when;
public class TestHoodieMetrics {
- private HoodieMetrics metrics = null;
-
@Before
public void start() {
HoodieWriteConfig config = mock(HoodieWriteConfig.class);
when(config.isMetricsOn()).thenReturn(true);
when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.INMEMORY);
- metrics = new HoodieMetrics(config, "raw_table");
+ new HoodieMetrics(config, "raw_table");
}
@Test
diff --git a/pom.xml b/pom.xml
index aa4d706..c61e4a6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -83,7 +83,7 @@
<hadoop.version>2.7.3</hadoop.version>
<hive.groupid>org.apache.hive</hive.groupid>
<hive.version>2.3.1</hive.version>
- <metrics.version>4.0.2</metrics.version>
+ <metrics.version>4.1.1</metrics.version>
<spark.version>2.1.0</spark.version>
<avro.version>1.7.7</avro.version>
<scala.version>2.11.8</scala.version>
