This is an automated email from the ASF dual-hosted git repository.
vinoyang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 2791fb9 [HUDI-2423] Separate some config logic from
HoodieMetricsConfig into HoodieMetricsGraphiteConfig HoodieMetricsJmxConfig
(#3652)
2791fb9 is described below
commit 2791fb9a964b39ef9aaec83eafd080013186b2eb
Author: liujinhui <[email protected]>
AuthorDate: Thu Sep 16 15:08:10 2021 +0800
[HUDI-2423] Separate some config logic from HoodieMetricsConfig into
HoodieMetricsGraphiteConfig HoodieMetricsJmxConfig (#3652)
---
.../org/apache/hudi/config/HoodieWriteConfig.java | 29 ++++-
.../config/{ => metrics}/HoodieMetricsConfig.java | 112 +----------------
.../{ => metrics}/HoodieMetricsDatadogConfig.java | 4 +-
.../metrics/HoodieMetricsGraphiteConfig.java | 134 +++++++++++++++++++++
.../config/metrics/HoodieMetricsJmxConfig.java | 118 ++++++++++++++++++
.../HoodieMetricsPrometheusConfig.java | 45 ++++++-
.../metadata/HoodieBackedTableMetadataWriter.java | 22 ++--
.../datadog/TestHoodieMetricsDatadogConfig.java | 2 +-
.../functional/TestHoodieBackedMetadata.java | 7 +-
9 files changed, 344 insertions(+), 129 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index 4df7d0d..c871253 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -40,6 +40,11 @@ import
org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.common.util.ValidationUtils;
+import org.apache.hudi.config.metrics.HoodieMetricsConfig;
+import org.apache.hudi.config.metrics.HoodieMetricsDatadogConfig;
+import org.apache.hudi.config.metrics.HoodieMetricsGraphiteConfig;
+import org.apache.hudi.config.metrics.HoodieMetricsJmxConfig;
+import org.apache.hudi.config.metrics.HoodieMetricsPrometheusConfig;
import org.apache.hudi.execution.bulkinsert.BulkInsertSortMode;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.keygen.SimpleAvroKeyGenerator;
@@ -1459,23 +1464,23 @@ public class HoodieWriteConfig extends HoodieConfig {
}
public String getGraphiteServerHost() {
- return getString(HoodieMetricsConfig.GRAPHITE_SERVER_HOST_NAME);
+ return getString(HoodieMetricsGraphiteConfig.GRAPHITE_SERVER_HOST_NAME);
}
public int getGraphiteServerPort() {
- return getInt(HoodieMetricsConfig.GRAPHITE_SERVER_PORT_NUM);
+ return getInt(HoodieMetricsGraphiteConfig.GRAPHITE_SERVER_PORT_NUM);
}
public String getGraphiteMetricPrefix() {
- return getString(HoodieMetricsConfig.GRAPHITE_METRIC_PREFIX_VALUE);
+ return getString(HoodieMetricsGraphiteConfig.GRAPHITE_METRIC_PREFIX_VALUE);
}
public String getJmxHost() {
- return getString(HoodieMetricsConfig.JMX_HOST_NAME);
+ return getString(HoodieMetricsJmxConfig.JMX_HOST_NAME);
}
public String getJmxPort() {
- return getString(HoodieMetricsConfig.JMX_PORT_NUM);
+ return getString(HoodieMetricsJmxConfig.JMX_PORT_NUM);
}
public int getDatadogReportPeriodSeconds() {
@@ -1777,6 +1782,8 @@ public class HoodieWriteConfig extends HoodieConfig {
private boolean isMetadataConfigSet = false;
private boolean isLockConfigSet = false;
private boolean isPreCommitValidationConfigSet = false;
+ private boolean isMetricsJmxConfigSet = false;
+ private boolean isMetricsGraphiteConfigSet = false;
public Builder withEngineType(EngineType engineType) {
this.engineType = engineType;
@@ -1931,6 +1938,18 @@ public class HoodieWriteConfig extends HoodieConfig {
return this;
}
+ public Builder withMetricsJmxConfig(HoodieMetricsJmxConfig
metricsJmxConfig) {
+ writeConfig.getProps().putAll(metricsJmxConfig.getProps());
+ isMetricsJmxConfigSet = true;
+ return this;
+ }
+
+ public Builder withMetricsGraphiteConfig(HoodieMetricsGraphiteConfig
mericsGraphiteConfig) {
+ writeConfig.getProps().putAll(mericsGraphiteConfig.getProps());
+ isMetricsGraphiteConfigSet = true;
+ return this;
+ }
+
public Builder withPreCommitValidatorConfig(HoodiePreCommitValidatorConfig
validatorConfig) {
writeConfig.getProps().putAll(validatorConfig.getProps());
isPreCommitValidationConfigSet = true;
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java
similarity index 62%
rename from
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java
rename to
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java
index e1d0b63..60369d7 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsConfig.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.config;
+package org.apache.hudi.config.metrics;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
@@ -55,42 +55,6 @@ public class HoodieMetricsConfig extends HoodieConfig {
.sinceVersion("0.5.0")
.withDocumentation("Type of metrics reporter.");
- // Graphite
- public static final String GRAPHITE_PREFIX = METRIC_PREFIX + ".graphite";
-
- public static final ConfigProperty<String> GRAPHITE_SERVER_HOST_NAME =
ConfigProperty
- .key(GRAPHITE_PREFIX + ".host")
- .defaultValue("localhost")
- .sinceVersion("0.5.0")
- .withDocumentation("Graphite host to connect to");
-
- public static final ConfigProperty<Integer> GRAPHITE_SERVER_PORT_NUM =
ConfigProperty
- .key(GRAPHITE_PREFIX + ".port")
- .defaultValue(4756)
- .sinceVersion("0.5.0")
- .withDocumentation("Graphite port to connect to");
-
- // Jmx
- public static final String JMX_PREFIX = METRIC_PREFIX + ".jmx";
-
- public static final ConfigProperty<String> JMX_HOST_NAME = ConfigProperty
- .key(JMX_PREFIX + ".host")
- .defaultValue("localhost")
- .sinceVersion("0.5.1")
- .withDocumentation("Jmx host to connect to");
-
- public static final ConfigProperty<Integer> JMX_PORT_NUM = ConfigProperty
- .key(JMX_PREFIX + ".port")
- .defaultValue(9889)
- .sinceVersion("0.5.1")
- .withDocumentation("Jmx port to connect to");
-
- public static final ConfigProperty<String> GRAPHITE_METRIC_PREFIX_VALUE =
ConfigProperty
- .key(GRAPHITE_PREFIX + ".metric.prefix")
- .noDefaultValue()
- .sinceVersion("0.5.1")
- .withDocumentation("Standard prefix applied to all metrics. This helps
to add datacenter, environment information for e.g");
-
// User defined
public static final ConfigProperty<String> METRICS_REPORTER_CLASS_NAME =
ConfigProperty
.key(METRIC_PREFIX + ".reporter.class")
@@ -126,51 +90,6 @@ public class HoodieMetricsConfig extends HoodieConfig {
@Deprecated
public static final MetricsReporterType DEFAULT_METRICS_REPORTER_TYPE =
METRICS_REPORTER_TYPE_VALUE.defaultValue();
/**
- * @deprecated Use {@link #GRAPHITE_SERVER_HOST_NAME} and its methods instead
- */
- @Deprecated
- public static final String GRAPHITE_SERVER_HOST =
GRAPHITE_SERVER_HOST_NAME.key();
- /**
- * @deprecated Use {@link #GRAPHITE_SERVER_HOST_NAME} and its methods instead
- */
- @Deprecated
- public static final String DEFAULT_GRAPHITE_SERVER_HOST =
GRAPHITE_SERVER_HOST_NAME.defaultValue();
- /**
- * @deprecated Use {@link #GRAPHITE_SERVER_PORT_NUM} and its methods instead
- */
- @Deprecated
- public static final String GRAPHITE_SERVER_PORT =
GRAPHITE_SERVER_PORT_NUM.key();
- /**
- * @deprecated Use {@link #GRAPHITE_SERVER_PORT_NUM} and its methods instead
- */
- @Deprecated
- public static final int DEFAULT_GRAPHITE_SERVER_PORT =
GRAPHITE_SERVER_PORT_NUM.defaultValue();
- /**
- * @deprecated Use {@link #JMX_HOST_NAME} and its methods instead
- */
- @Deprecated
- public static final String JMX_HOST = JMX_HOST_NAME.key();
- /**
- * @deprecated Use {@link #JMX_HOST_NAME} and its methods instead
- */
- @Deprecated
- public static final String DEFAULT_JMX_HOST = JMX_HOST_NAME.defaultValue();
- /**
- * @deprecated Use {@link #JMX_PORT_NUM} and its methods instead
- */
- @Deprecated
- public static final String JMX_PORT = JMX_PORT_NUM.key();
- /**
- * @deprecated Use {@link #JMX_PORT_NUM} and its methods instead
- */
- @Deprecated
- public static final int DEFAULT_JMX_PORT = JMX_PORT_NUM.defaultValue();
- /**
- * @deprecated Use {@link #GRAPHITE_METRIC_PREFIX_VALUE} and its methods
instead
- */
- @Deprecated
- public static final String GRAPHITE_METRIC_PREFIX =
GRAPHITE_METRIC_PREFIX_VALUE.key();
- /**
* @deprecated Use {@link #METRICS_REPORTER_CLASS_NAME} and its methods
instead
*/
@Deprecated
@@ -220,31 +139,6 @@ public class HoodieMetricsConfig extends HoodieConfig {
return this;
}
- public Builder toGraphiteHost(String host) {
- hoodieMetricsConfig.setValue(GRAPHITE_SERVER_HOST_NAME, host);
- return this;
- }
-
- public Builder onGraphitePort(int port) {
- hoodieMetricsConfig.setValue(GRAPHITE_SERVER_PORT_NUM,
String.valueOf(port));
- return this;
- }
-
- public Builder toJmxHost(String host) {
- hoodieMetricsConfig.setValue(JMX_HOST_NAME, host);
- return this;
- }
-
- public Builder onJmxPort(String port) {
- hoodieMetricsConfig.setValue(JMX_PORT_NUM, port);
- return this;
- }
-
- public Builder usePrefix(String prefix) {
- hoodieMetricsConfig.setValue(GRAPHITE_METRIC_PREFIX_VALUE, prefix);
- return this;
- }
-
public Builder withReporterClass(String className) {
hoodieMetricsConfig.setValue(METRICS_REPORTER_CLASS_NAME, className);
return this;
@@ -267,6 +161,10 @@ public class HoodieMetricsConfig extends HoodieConfig {
HoodieMetricsPrometheusConfig.newBuilder().fromProperties(hoodieMetricsConfig.getProps()).build());
hoodieMetricsConfig.setDefaultOnCondition(reporterType ==
MetricsReporterType.PROMETHEUS,
HoodieMetricsPrometheusConfig.newBuilder().fromProperties(hoodieMetricsConfig.getProps()).build());
+ hoodieMetricsConfig.setDefaultOnCondition(reporterType ==
MetricsReporterType.JMX,
+
HoodieMetricsJmxConfig.newBuilder().fromProperties(hoodieMetricsConfig.getProps()).build());
+ hoodieMetricsConfig.setDefaultOnCondition(reporterType ==
MetricsReporterType.GRAPHITE,
+
HoodieMetricsGraphiteConfig.newBuilder().fromProperties(hoodieMetricsConfig.getProps()).build());
return hoodieMetricsConfig;
}
}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsDatadogConfig.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsDatadogConfig.java
similarity index 98%
rename from
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsDatadogConfig.java
rename to
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsDatadogConfig.java
index d63cb0f..3fc306b 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsDatadogConfig.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsDatadogConfig.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.config;
+package org.apache.hudi.config.metrics;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
@@ -27,7 +27,7 @@ import javax.annotation.concurrent.Immutable;
import java.util.Properties;
-import static org.apache.hudi.config.HoodieMetricsConfig.METRIC_PREFIX;
+import static org.apache.hudi.config.metrics.HoodieMetricsConfig.METRIC_PREFIX;
/**
* Configs for Datadog reporter type.
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsGraphiteConfig.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsGraphiteConfig.java
new file mode 100644
index 0000000..12987a7
--- /dev/null
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsGraphiteConfig.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.config.metrics;
+
+import org.apache.hudi.common.config.ConfigClassProperty;
+import org.apache.hudi.common.config.ConfigGroups;
+import org.apache.hudi.common.config.ConfigProperty;
+import org.apache.hudi.common.config.HoodieConfig;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.Properties;
+
+import static org.apache.hudi.config.metrics.HoodieMetricsConfig.METRIC_PREFIX;
+
+/**
+ * Configs for Graphite reporter type.
+ * <p>
+ * {@link org.apache.hudi.metrics.MetricsReporterType#GRAPHITE}
+ */
+@ConfigClassProperty(name = "Metrics Configurations for Graphite",
+ groupName = ConfigGroups.Names.METRICS,
+ description = "Enables reporting on Hudi metrics using Graphite. "
+ + " Hudi publishes metrics on every commit, clean, rollback etc.")
+public class HoodieMetricsGraphiteConfig extends HoodieConfig {
+
+ public static final String GRAPHITE_PREFIX = METRIC_PREFIX + ".graphite";
+
+ public static final ConfigProperty<String> GRAPHITE_SERVER_HOST_NAME =
ConfigProperty
+ .key(GRAPHITE_PREFIX + ".host")
+ .defaultValue("localhost")
+ .sinceVersion("0.5.0")
+ .withDocumentation("Graphite host to connect to.");
+
+ public static final ConfigProperty<Integer> GRAPHITE_SERVER_PORT_NUM =
ConfigProperty
+ .key(GRAPHITE_PREFIX + ".port")
+ .defaultValue(4756)
+ .sinceVersion("0.5.0")
+ .withDocumentation("Graphite port to connect to.");
+
+ public static final ConfigProperty<String> GRAPHITE_METRIC_PREFIX_VALUE =
ConfigProperty
+ .key(GRAPHITE_PREFIX + ".metric.prefix")
+ .noDefaultValue()
+ .sinceVersion("0.5.1")
+ .withDocumentation("Standard prefix applied to all metrics. This helps
to add datacenter, environment information for e.g");
+
+ /**
+ * @deprecated Use {@link #GRAPHITE_SERVER_HOST_NAME} and its methods instead
+ */
+ @Deprecated
+ public static final String GRAPHITE_SERVER_HOST =
GRAPHITE_SERVER_HOST_NAME.key();
+ /**
+ * @deprecated Use {@link #GRAPHITE_SERVER_HOST_NAME} and its methods instead
+ */
+ @Deprecated
+ public static final String DEFAULT_GRAPHITE_SERVER_HOST =
GRAPHITE_SERVER_HOST_NAME.defaultValue();
+ /**
+ * @deprecated Use {@link #GRAPHITE_SERVER_PORT_NUM} and its methods instead
+ */
+ @Deprecated
+ public static final String GRAPHITE_SERVER_PORT =
GRAPHITE_SERVER_PORT_NUM.key();
+ /**
+ * @deprecated Use {@link #GRAPHITE_SERVER_PORT_NUM} and its methods instead
+ */
+ @Deprecated
+ public static final int DEFAULT_GRAPHITE_SERVER_PORT =
GRAPHITE_SERVER_PORT_NUM.defaultValue();
+ /**
+ * @deprecated Use {@link #GRAPHITE_METRIC_PREFIX_VALUE} and its methods
instead
+ */
+ @Deprecated
+ public static final String GRAPHITE_METRIC_PREFIX =
GRAPHITE_METRIC_PREFIX_VALUE.key();
+
+ private HoodieMetricsGraphiteConfig() {
+ super();
+ }
+
+ public static HoodieMetricsGraphiteConfig.Builder newBuilder() {
+ return new HoodieMetricsGraphiteConfig.Builder();
+ }
+
+ public static class Builder {
+
+ private final HoodieMetricsGraphiteConfig hoodieMetricsGraphiteConfig =
new HoodieMetricsGraphiteConfig();
+
+ public HoodieMetricsGraphiteConfig.Builder fromFile(File propertiesFile)
throws IOException {
+ try (FileReader reader = new FileReader(propertiesFile)) {
+ this.hoodieMetricsGraphiteConfig.getProps().load(reader);
+ return this;
+ }
+ }
+
+ public HoodieMetricsGraphiteConfig.Builder fromProperties(Properties
props) {
+ this.hoodieMetricsGraphiteConfig.getProps().putAll(props);
+ return this;
+ }
+
+ public HoodieMetricsGraphiteConfig.Builder toGraphiteHost(String host) {
+ hoodieMetricsGraphiteConfig.setValue(GRAPHITE_SERVER_HOST_NAME, host);
+ return this;
+ }
+
+ public HoodieMetricsGraphiteConfig.Builder onGraphitePort(int port) {
+ hoodieMetricsGraphiteConfig.setValue(GRAPHITE_SERVER_PORT_NUM,
String.valueOf(port));
+ return this;
+ }
+
+ public HoodieMetricsGraphiteConfig.Builder usePrefix(String prefix) {
+ hoodieMetricsGraphiteConfig.setValue(GRAPHITE_METRIC_PREFIX_VALUE,
prefix);
+ return this;
+ }
+
+ public HoodieMetricsGraphiteConfig build() {
+
hoodieMetricsGraphiteConfig.setDefaults(HoodieMetricsGraphiteConfig.class.getName());
+ return hoodieMetricsGraphiteConfig;
+ }
+ }
+}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsJmxConfig.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsJmxConfig.java
new file mode 100644
index 0000000..e3a57a1
--- /dev/null
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsJmxConfig.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.config.metrics;
+
+import org.apache.hudi.common.config.ConfigClassProperty;
+import org.apache.hudi.common.config.ConfigGroups;
+import org.apache.hudi.common.config.ConfigProperty;
+import org.apache.hudi.common.config.HoodieConfig;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.Properties;
+
+import static org.apache.hudi.config.metrics.HoodieMetricsConfig.METRIC_PREFIX;
+
+/**
+ * Configs for Jmx reporter type.
+ * <p>
+ * {@link org.apache.hudi.metrics.MetricsReporterType#JMX}
+ */
+@ConfigClassProperty(name = "Metrics Configurations for Jmx",
+ groupName = ConfigGroups.Names.METRICS,
+ description = "Enables reporting on Hudi metrics using Jmx. "
+ + " Hudi publishes metrics on every commit, clean,
rollback etc.")
+public class HoodieMetricsJmxConfig extends HoodieConfig {
+
+ public static final String JMX_PREFIX = METRIC_PREFIX + ".jmx";
+
+ public static final ConfigProperty<String> JMX_HOST_NAME = ConfigProperty
+ .key(JMX_PREFIX + ".host")
+ .defaultValue("localhost")
+ .sinceVersion("0.5.1")
+ .withDocumentation("Jmx host to connect to");
+
+ public static final ConfigProperty<Integer> JMX_PORT_NUM = ConfigProperty
+ .key(JMX_PREFIX + ".port")
+ .defaultValue(9889)
+ .sinceVersion("0.5.1")
+ .withDocumentation("Jmx port to connect to");
+
+ /**
+ * @deprecated Use {@link #JMX_HOST_NAME} and its methods instead
+ */
+ @Deprecated
+ public static final String JMX_HOST = JMX_HOST_NAME.key();
+ /**
+ * @deprecated Use {@link #JMX_HOST_NAME} and its methods instead
+ */
+ @Deprecated
+ public static final String DEFAULT_JMX_HOST = JMX_HOST_NAME.defaultValue();
+ /**
+ * @deprecated Use {@link #JMX_PORT_NUM} and its methods instead
+ */
+ @Deprecated
+ public static final String JMX_PORT = JMX_PORT_NUM.key();
+ /**
+ * @deprecated Use {@link #JMX_PORT_NUM} and its methods instead
+ */
+ @Deprecated
+ public static final int DEFAULT_JMX_PORT = JMX_PORT_NUM.defaultValue();
+
+ private HoodieMetricsJmxConfig() {
+ super();
+ }
+
+ public static HoodieMetricsJmxConfig.Builder newBuilder() {
+ return new HoodieMetricsJmxConfig.Builder();
+ }
+
+ public static class Builder {
+
+ private final HoodieMetricsJmxConfig hoodieMetricsJmxConfig = new
HoodieMetricsJmxConfig();
+
+ public HoodieMetricsJmxConfig.Builder fromFile(File propertiesFile) throws
IOException {
+ try (FileReader reader = new FileReader(propertiesFile)) {
+ this.hoodieMetricsJmxConfig.getProps().load(reader);
+ return this;
+ }
+ }
+
+ public HoodieMetricsJmxConfig.Builder fromProperties(Properties props) {
+ this.hoodieMetricsJmxConfig.getProps().putAll(props);
+ return this;
+ }
+
+ public HoodieMetricsJmxConfig.Builder toJmxHost(String host) {
+ hoodieMetricsJmxConfig.setValue(JMX_HOST_NAME, host);
+ return this;
+ }
+
+ public HoodieMetricsJmxConfig.Builder onJmxPort(String port) {
+ hoodieMetricsJmxConfig.setValue(JMX_PORT_NUM, port);
+ return this;
+ }
+
+ public HoodieMetricsJmxConfig build() {
+
hoodieMetricsJmxConfig.setDefaults(HoodieMetricsJmxConfig.class.getName());
+ return hoodieMetricsJmxConfig;
+ }
+ }
+}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsPrometheusConfig.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsPrometheusConfig.java
similarity index 78%
rename from
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsPrometheusConfig.java
rename to
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsPrometheusConfig.java
index 9ec2ec6..c04e8aa 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieMetricsPrometheusConfig.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/metrics/HoodieMetricsPrometheusConfig.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.config;
+package org.apache.hudi.config.metrics;
import org.apache.hudi.common.config.ConfigClassProperty;
import org.apache.hudi.common.config.ConfigGroups;
@@ -25,8 +25,14 @@ import org.apache.hudi.common.config.HoodieConfig;
import java.util.Properties;
-import static org.apache.hudi.config.HoodieMetricsConfig.METRIC_PREFIX;
+import static org.apache.hudi.config.metrics.HoodieMetricsConfig.METRIC_PREFIX;
+/**
+ * Configs for Prometheus/Pushgaeway reporter type.
+ * <p>
+ * {@link org.apache.hudi.metrics.MetricsReporterType#PROMETHEUS}
+ * {@link org.apache.hudi.metrics.MetricsReporterType#PROMETHEUS_PUSHGATEWAY}
+ */
@ConfigClassProperty(name = "Metrics Configurations for Prometheus",
groupName = ConfigGroups.Names.METRICS,
description = "Enables reporting on Hudi metrics using Prometheus. "
@@ -174,6 +180,41 @@ public class HoodieMetricsPrometheusConfig extends
HoodieConfig {
return this;
}
+ public HoodieMetricsPrometheusConfig.Builder
withPushgatewayHostName(String hostName) {
+ hoodieMetricsPrometheusConfig.setValue(PUSHGATEWAY_HOST_NAME,
String.valueOf(hostName));
+ return this;
+ }
+
+ public HoodieMetricsPrometheusConfig.Builder
withPushgatewayPortNum(Integer pushgatewayPortNum) {
+ hoodieMetricsPrometheusConfig.setValue(PUSHGATEWAY_PORT_NUM,
String.valueOf(pushgatewayPortNum));
+ return this;
+ }
+
+ public HoodieMetricsPrometheusConfig.Builder
withPushgatewayReportPeriodInSeconds(String periodTime) {
+
hoodieMetricsPrometheusConfig.setValue(PUSHGATEWAY_REPORT_PERIOD_IN_SECONDS,
periodTime);
+ return this;
+ }
+
+ public HoodieMetricsPrometheusConfig.Builder
withPushgatewayDeleteOnShutdownEnable(boolean deleteOnShutdownEnable) {
+
hoodieMetricsPrometheusConfig.setValue(PUSHGATEWAY_DELETE_ON_SHUTDOWN_ENABLE,
String.valueOf(deleteOnShutdownEnable));
+ return this;
+ }
+
+ public HoodieMetricsPrometheusConfig.Builder withPushgatewayJobname(String
jobname) {
+ hoodieMetricsPrometheusConfig.setValue(PUSHGATEWAY_JOBNAME, jobname);
+ return this;
+ }
+
+ public HoodieMetricsPrometheusConfig.Builder
withPushgatewayRandomJobnameSuffix(boolean randomJobnameSuffix) {
+
hoodieMetricsPrometheusConfig.setValue(PUSHGATEWAY_RANDOM_JOBNAME_SUFFIX,
String.valueOf(randomJobnameSuffix));
+ return this;
+ }
+
+ public HoodieMetricsPrometheusConfig.Builder withPrometheusPortNum(int
prometheusPortNum) {
+ hoodieMetricsPrometheusConfig.setValue(PROMETHEUS_PORT_NUM,
String.valueOf(prometheusPortNum));
+ return this;
+ }
+
public HoodieMetricsPrometheusConfig build() {
hoodieMetricsPrometheusConfig.setDefaults(HoodieMetricsPrometheusConfig.class.getName());
return hoodieMetricsPrometheusConfig;
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index ab5fa99..f797945 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -46,8 +46,10 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieCompactionConfig;
-import org.apache.hudi.config.HoodieMetricsConfig;
+import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.config.metrics.HoodieMetricsGraphiteConfig;
+import org.apache.hudi.config.metrics.HoodieMetricsJmxConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieMetadataException;
@@ -175,19 +177,22 @@ public abstract class HoodieBackedTableMetadataWriter
implements HoodieTableMeta
.withFinalizeWriteParallelism(parallelism);
if (writeConfig.isMetricsOn()) {
- HoodieMetricsConfig.Builder metricsConfig =
HoodieMetricsConfig.newBuilder()
+ builder.withMetricsConfig(HoodieMetricsConfig.newBuilder()
.withReporterType(writeConfig.getMetricsReporterType().toString())
.withExecutorMetrics(writeConfig.isExecutorMetricsEnabled())
- .on(true);
+ .on(true).build());
switch (writeConfig.getMetricsReporterType()) {
case GRAPHITE:
- metricsConfig.onGraphitePort(writeConfig.getGraphiteServerPort())
+
builder.withMetricsGraphiteConfig(HoodieMetricsGraphiteConfig.newBuilder()
+ .onGraphitePort(writeConfig.getGraphiteServerPort())
.toGraphiteHost(writeConfig.getGraphiteServerHost())
- .usePrefix(writeConfig.getGraphiteMetricPrefix());
+ .usePrefix(writeConfig.getGraphiteMetricPrefix()).build());
break;
case JMX:
- metricsConfig.onJmxPort(writeConfig.getJmxPort())
- .toJmxHost(writeConfig.getJmxHost());
+ builder.withMetricsJmxConfig(HoodieMetricsJmxConfig.newBuilder()
+ .onJmxPort(writeConfig.getJmxPort())
+ .toJmxHost(writeConfig.getJmxHost())
+ .build());
break;
case DATADOG:
case PROMETHEUS:
@@ -198,10 +203,7 @@ public abstract class HoodieBackedTableMetadataWriter
implements HoodieTableMeta
default:
throw new HoodieMetadataException("Unsupported Metrics Reporter type
" + writeConfig.getMetricsReporterType());
}
-
- builder.withMetricsConfig(metricsConfig.build());
}
-
return builder.build();
}
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestHoodieMetricsDatadogConfig.java
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestHoodieMetricsDatadogConfig.java
index 016e070..aa486e9 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestHoodieMetricsDatadogConfig.java
+++
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestHoodieMetricsDatadogConfig.java
@@ -19,7 +19,7 @@
package org.apache.hudi.metrics.datadog;
-import org.apache.hudi.config.HoodieMetricsDatadogConfig;
+import org.apache.hudi.config.metrics.HoodieMetricsDatadogConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.junit.jupiter.api.Test;
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
index bec7ee4..b091359 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
@@ -47,9 +47,10 @@ import org.apache.hudi.common.util.HoodieTimer;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieIndexConfig;
-import org.apache.hudi.config.HoodieMetricsConfig;
+import org.apache.hudi.config.metrics.HoodieMetricsConfig;
import org.apache.hudi.config.HoodieStorageConfig;
import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.config.metrics.HoodieMetricsGraphiteConfig;
import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.exception.TableNotFoundException;
import org.apache.hudi.index.HoodieIndex;
@@ -1305,7 +1306,9 @@ public class TestHoodieBackedMetadata extends
HoodieClientTestHarness {
.enable(useFileListingMetadata)
.enableMetrics(enableMetrics).build())
.withMetricsConfig(HoodieMetricsConfig.newBuilder().on(enableMetrics)
- .withExecutorMetrics(true).usePrefix("unit-test").build());
+ .withExecutorMetrics(true).build())
+ .withMetricsGraphiteConfig(HoodieMetricsGraphiteConfig.newBuilder()
+ .usePrefix("unit-test").build());
}
@Override
