This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 3418a92 [HUDI-1620] Fix Metrics UT (#2894)
3418a92 is described below
commit 3418a92de8a45986e55a29a9498791222f6a6fa6
Author: Raymond Xu <[email protected]>
AuthorDate: Fri Apr 30 11:20:41 2021 -0700
[HUDI-1620] Fix Metrics UT (#2894)
Make sure shutdown Metrics between unit test cases to ensure isolation
---
azure-pipelines.yml | 6 ++--
.../hudi/metrics/TestHoodieConsoleMetrics.java | 8 +++--
.../apache/hudi/metrics/TestHoodieJmxMetrics.java | 16 ++++++++--
.../org/apache/hudi/metrics/TestHoodieMetrics.java | 17 +++++++++--
.../datadog/TestDatadogMetricsReporter.java | 7 +++++
.../metrics/prometheus/TestPrometheusReporter.java | 18 ++++++++++--
.../prometheus/TestPushGateWayReporter.java | 28 +++++++++++++++---
pom.xml | 34 ----------------------
8 files changed, 83 insertions(+), 51 deletions(-)
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index e75e1b3..cf2343d 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -56,7 +56,7 @@ stages:
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
- options: -Pazp-unit-tests -pl hudi-client/hudi-spark-client
+ options: -Punit-tests -pl hudi-client/hudi-spark-client
publishJUnitResults: false
testResultsFiles: '**/surefire-reports/TEST-*.xml'
testRunTitle: 'unit tests spark client'
@@ -88,7 +88,7 @@ stages:
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
- options: -Pazp-unit-tests -pl hudi-utilities
+ options: -Punit-tests -pl hudi-utilities
publishJUnitResults: false
testResultsFiles: '**/surefire-reports/TEST-*.xml'
testRunTitle: 'unit tests utilities'
@@ -120,7 +120,7 @@ stages:
inputs:
mavenPomFile: 'pom.xml'
goals: 'test'
- options: -Pazp-unit-tests -pl
!hudi-utilities,!hudi-client/hudi-spark-client
+ options: -Punit-tests -pl
!hudi-utilities,!hudi-client/hudi-spark-client
publishJUnitResults: false
testResultsFiles: '**/surefire-reports/TEST-*.xml'
testRunTitle: 'unit tests other modules'
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieConsoleMetrics.java
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieConsoleMetrics.java
index 7424d0b..8c83562 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieConsoleMetrics.java
+++
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieConsoleMetrics.java
@@ -22,15 +22,19 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
import static org.apache.hudi.metrics.Metrics.registerGauge;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+@ExtendWith(MockitoExtension.class)
public class TestHoodieConsoleMetrics {
- HoodieWriteConfig config = mock(HoodieWriteConfig.class);
+ @Mock
+ HoodieWriteConfig config;
@BeforeEach
public void start() {
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java
index 7b63a30..8254bf6 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java
+++
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java
@@ -21,23 +21,34 @@ package org.apache.hudi.metrics;
import org.apache.hudi.common.testutils.NetworkTestUtils;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
import static org.apache.hudi.metrics.Metrics.registerGauge;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
/**
* Test for the Jmx metrics report.
*/
+@ExtendWith(MockitoExtension.class)
public class TestHoodieJmxMetrics {
- HoodieWriteConfig config = mock(HoodieWriteConfig.class);
+ @Mock
+ HoodieWriteConfig config;
+
+ @AfterEach
+ void shutdownMetrics() {
+ Metrics.shutdown();
+ }
@Test
public void testRegisterGauge() {
when(config.isMetricsOn()).thenReturn(true);
+ when(config.getTableName()).thenReturn("foo");
when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.JMX);
when(config.getJmxHost()).thenReturn("localhost");
when(config.getJmxPort()).thenReturn(String.valueOf(NetworkTestUtils.nextFreePort()));
@@ -50,6 +61,7 @@ public class TestHoodieJmxMetrics {
@Test
public void testRegisterGaugeByRangerPort() {
when(config.isMetricsOn()).thenReturn(true);
+ when(config.getTableName()).thenReturn("foo");
when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.JMX);
when(config.getJmxHost()).thenReturn("localhost");
when(config.getJmxPort()).thenReturn(String.valueOf(NetworkTestUtils.nextFreePort()));
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
index e669a67..a734b8c 100755
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
+++
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java
@@ -24,8 +24,12 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import com.codahale.metrics.Timer;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
import java.util.Random;
import java.util.stream.Stream;
@@ -36,18 +40,25 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+@ExtendWith(MockitoExtension.class)
public class TestHoodieMetrics {
- private HoodieMetrics metrics;
+ @Mock
+ HoodieWriteConfig config;
+ HoodieMetrics metrics;
@BeforeEach
- public void start() {
- HoodieWriteConfig config = mock(HoodieWriteConfig.class);
+ void setUp() {
when(config.isMetricsOn()).thenReturn(true);
when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.INMEMORY);
metrics = new HoodieMetrics(config, "raw_table");
}
+ @AfterEach
+ void shutdownMetrics() {
+ Metrics.shutdown();
+ }
+
@Test
public void testRegisterGauge() {
registerGauge("metric1", 123L);
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogMetricsReporter.java
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogMetricsReporter.java
index 3cab8f6..2514a48 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogMetricsReporter.java
+++
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogMetricsReporter.java
@@ -19,9 +19,11 @@
package org.apache.hudi.metrics.datadog;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.metrics.Metrics;
import org.apache.hudi.metrics.datadog.DatadogHttpClient.ApiSite;
import com.codahale.metrics.MetricRegistry;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
@@ -43,6 +45,11 @@ public class TestDatadogMetricsReporter {
@Mock
MetricRegistry registry;
+ @AfterEach
+ void shutdownMetrics() {
+ Metrics.shutdown();
+ }
+
@Test
public void instantiationShouldFailWhenNoApiKey() {
when(config.getDatadogApiKey()).thenReturn("");
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPrometheusReporter.java
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPrometheusReporter.java
index 6bbd49d..9010fe5 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPrometheusReporter.java
+++
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPrometheusReporter.java
@@ -20,16 +20,28 @@ package org.apache.hudi.metrics.prometheus;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.metrics.HoodieMetrics;
+import org.apache.hudi.metrics.Metrics;
import org.apache.hudi.metrics.MetricsReporterType;
+
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
-import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+@ExtendWith(MockitoExtension.class)
public class TestPrometheusReporter {
- HoodieWriteConfig config = mock(HoodieWriteConfig.class);
+ @Mock
+ HoodieWriteConfig config;
+
+ @AfterEach
+ void shutdownMetrics() {
+ Metrics.shutdown();
+ }
@Test
public void testRegisterGauge() {
@@ -40,4 +52,4 @@ public class TestPrometheusReporter {
new HoodieMetrics(config, "raw_table");
});
}
-}
\ No newline at end of file
+}
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPushGateWayReporter.java
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPushGateWayReporter.java
index 2b94226..eb4d09c 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPushGateWayReporter.java
+++
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPushGateWayReporter.java
@@ -22,16 +22,28 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.metrics.HoodieMetrics;
import org.apache.hudi.metrics.Metrics;
import org.apache.hudi.metrics.MetricsReporterType;
+
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
import static org.apache.hudi.metrics.Metrics.registerGauge;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+@ExtendWith(MockitoExtension.class)
public class TestPushGateWayReporter {
- HoodieWriteConfig config = mock(HoodieWriteConfig.class);
+ @Mock
+ HoodieWriteConfig config;
+
+ @AfterEach
+ void shutdownMetrics() {
+ Metrics.shutdown();
+ }
@Test
public void testRegisterGauge() {
@@ -39,9 +51,17 @@ public class TestPushGateWayReporter {
when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.PROMETHEUS_PUSHGATEWAY);
when(config.getPushGatewayHost()).thenReturn("localhost");
when(config.getPushGatewayPort()).thenReturn(9091);
- new HoodieMetrics(config, "raw_table");
+ when(config.getPushGatewayReportPeriodSeconds()).thenReturn(30);
+ when(config.getPushGatewayDeleteOnShutdown()).thenReturn(true);
+ when(config.getPushGatewayJobName()).thenReturn("foo");
+ when(config.getPushGatewayRandomJobNameSuffix()).thenReturn(false);
+
+ assertDoesNotThrow(() -> {
+ new HoodieMetrics(config, "raw_table");
+ });
+
registerGauge("pushGateWayReporter_metric", 123L);
assertEquals("123", Metrics.getInstance().getRegistry().getGauges()
.get("pushGateWayReporter_metric").getValue().toString());
}
-}
\ No newline at end of file
+}
diff --git a/pom.xml b/pom.xml
index 7f371dd..8a9a993 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1274,40 +1274,6 @@
</build>
</profile>
<profile>
- <id>azp-unit-tests</id>
- <properties>
- <skipUTs>false</skipUTs>
- <skipFTs>true</skipFTs>
- <skipITs>true</skipITs>
- </properties>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <dependencies>
- <dependency>
- <groupId>org.junit.jupiter</groupId>
- <artifactId>junit-jupiter-engine</artifactId>
- <version>${junit.jupiter.version}</version>
- </dependency>
- </dependencies>
- <configuration combine.self="append">
- <skip>${skipUTs}</skip>
-
<forkedProcessExitTimeoutInSeconds>120</forkedProcessExitTimeoutInSeconds>
- <excludedGroups>functional</excludedGroups>
- <excludes>
- <exclude>**/*FunctionalTestSuite.java</exclude>
- <exclude>**/IT*.java</exclude>
- <exclude>**/testsuite/**/Test*.java</exclude>
- <exclude>**/TestPushGateWayReporter.java</exclude>
- </excludes>
- </configuration>
- </plugin>
- </plugins>
- </build>
- </profile>
- <profile>
<id>javadocs</id>
<build>
<plugins>
