This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 6593b5f7ae1 [HUDI-8488] Introduce SLF4j metrics reporter (#12219)
6593b5f7ae1 is described below
commit 6593b5f7ae15e3fd7a5cc470fcf33b6366927898
Author: TheR1sing3un <[email protected]>
AuthorDate: Wed Nov 20 10:18:33 2024 +0800
[HUDI-8488] Introduce SLF4j metrics reporter (#12219)
1. introduce SLF4j metrics reporter
Signed-off-by: TheR1sing3un <[email protected]>
---
.../hudi/metrics/TestMetricsReporterFactory.java | 22 +++++--
.../org/apache/hudi/metrics/TestSlf4jMetrics.java | 70 +++++++++++++++++++++
.../hudi/metrics/MetricsReporterFactory.java | 3 +
.../apache/hudi/metrics/MetricsReporterType.java | 2 +-
.../apache/hudi/metrics/Slf4jMetricsReporter.java | 73 ++++++++++++++++++++++
5 files changed, 165 insertions(+), 5 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java
index bb2531011af..5afd27fbea3 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java
+++
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java
@@ -23,10 +23,14 @@ import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.metrics.custom.CustomizableMetricsReporter;
+import org.apache.hudi.metrics.prometheus.PrometheusReporter;
+import org.apache.hudi.metrics.prometheus.PushGatewayMetricsReporter;
import com.codahale.metrics.MetricRegistry;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
@@ -46,11 +50,21 @@ public class TestMetricsReporterFactory {
@Mock
MetricRegistry registry;
- @Test
- public void metricsReporterFactoryShouldReturnReporter() {
-
when(metricsConfig.getMetricsReporterType()).thenReturn(MetricsReporterType.INMEMORY);
+ public static Object[][] params() {
+ return new Object[][] {
+ {MetricsReporterType.INMEMORY, InMemoryMetricsReporter.class},
+ {MetricsReporterType.CONSOLE, ConsoleMetricsReporter.class},
+ {MetricsReporterType.PROMETHEUS, PrometheusReporter.class},
+ {MetricsReporterType.PROMETHEUS_PUSHGATEWAY,
PushGatewayMetricsReporter.class},
+ {MetricsReporterType.SLF4J, Slf4jMetricsReporter.class}};
+ }
+
+ @ParameterizedTest
+ @MethodSource("params")
+ public void metricsReporterFactoryShouldReturnReporter(MetricsReporterType
type, Class expectClazz) {
+ when(metricsConfig.getMetricsReporterType()).thenReturn(type);
MetricsReporter reporter =
MetricsReporterFactory.createReporter(metricsConfig, registry).get();
- assertTrue(reporter instanceof InMemoryMetricsReporter);
+ assertEquals(reporter.getClass(), expectClazz);
}
@Test
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestSlf4jMetrics.java
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestSlf4jMetrics.java
new file mode 100644
index 00000000000..6171f5f7030
--- /dev/null
+++
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestSlf4jMetrics.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.metrics;
+
+import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.config.metrics.HoodieMetricsConfig;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+public class TestSlf4jMetrics {
+
+ @Mock
+ HoodieWriteConfig writeConfig;
+ @Mock
+ HoodieMetricsConfig metricsConfig;
+ HoodieMetrics hoodieMetrics;
+ Metrics metrics;
+
+ @BeforeEach
+ public void start() {
+ when(writeConfig.getMetricsConfig()).thenReturn(metricsConfig);
+ when(writeConfig.isMetricsOn()).thenReturn(true);
+
when(metricsConfig.getMetricsReporterType()).thenReturn(MetricsReporterType.SLF4J);
+ when(metricsConfig.getBasePath()).thenReturn("s3://test" +
UUID.randomUUID());
+
when(metricsConfig.getMetricReporterMetricsNamePrefix()).thenReturn("hoodie");
+ hoodieMetrics = new HoodieMetrics(writeConfig,
HoodieTestUtils.getDefaultStorage());
+ metrics = hoodieMetrics.getMetrics();
+ }
+
+ @AfterEach
+ public void stop() {
+ metrics.shutdown();
+ }
+
+ @Test
+ public void testRegisterGauge() {
+ metrics.registerGauge("metric1", 123L);
+ assertEquals("123",
metrics.getRegistry().getGauges().get("metric1").getValue().toString());
+ metrics.flush();
+ }
+
+}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java
b/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java
index 455cf8de1c5..71509d7a9b7 100644
---
a/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java
+++
b/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java
@@ -92,6 +92,9 @@ public class MetricsReporterFactory {
case M3:
reporter = new M3MetricsReporter(metricsConfig, registry);
break;
+ case SLF4J:
+ reporter = new Slf4jMetricsReporter(registry);
+ break;
default:
LOG.error("Reporter type[" + type + "] is not supported.");
break;
diff --git
a/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterType.java
b/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterType.java
index 6d05e443e6b..b913577f3ab 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterType.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metrics/MetricsReporterType.java
@@ -22,5 +22,5 @@ package org.apache.hudi.metrics;
* Types of the reporter supported, hudi also supports user defined reporter.
*/
public enum MetricsReporterType {
- GRAPHITE, INMEMORY, JMX, DATADOG, CONSOLE, PROMETHEUS_PUSHGATEWAY,
PROMETHEUS, CLOUDWATCH, M3
+ GRAPHITE, INMEMORY, JMX, DATADOG, CONSOLE, PROMETHEUS_PUSHGATEWAY,
PROMETHEUS, CLOUDWATCH, M3, SLF4J
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/metrics/Slf4jMetricsReporter.java
b/hudi-common/src/main/java/org/apache/hudi/metrics/Slf4jMetricsReporter.java
new file mode 100644
index 00000000000..1a75f1ba358
--- /dev/null
+++
b/hudi-common/src/main/java/org/apache/hudi/metrics/Slf4jMetricsReporter.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.metrics;
+
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Slf4jReporter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Hudi Slf4j metrics reporter. Reports the metrics by printing them to the
log.
+ */
+public class Slf4jMetricsReporter extends MetricsReporter {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(Slf4jMetricsReporter.class);
+
+ private final Slf4jReporter reporter;
+
+ public Slf4jMetricsReporter(MetricRegistry registry) {
+ this.reporter = Slf4jReporter.forRegistry(registry)
+ .outputTo(LOG)
+ .convertRatesTo(TimeUnit.SECONDS)
+ .convertDurationsTo(TimeUnit.MILLISECONDS)
+ .filter(MetricFilter.ALL)
+ .build();
+ }
+
+ @Override
+ public void start() {
+ if (reporter != null) {
+ reporter.start(30, TimeUnit.SECONDS);
+ } else {
+ LOG.error("Cannot start as the reporter is null.");
+ }
+ }
+
+ @Override
+ public void report() {
+ if (reporter != null) {
+ reporter.report();
+ } else {
+ LOG.error("Cannot report metrics as the reporter is null.");
+ }
+ }
+
+ @Override
+ public void stop() {
+ if (reporter != null) {
+ reporter.stop();
+ } else {
+ LOG.error("Cannot stop as the reporter is null.");
+ }
+ }
+}
