This is an automated email from the ASF dual-hosted git repository.
nsivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new c6d707c6a8bd [MINOR] Guard detailed metadata size metrics with a
config (#18803)
c6d707c6a8bd is described below
commit c6d707c6a8bda4a18edab98247ebd03ff1530111
Author: Sivabalan Narayanan <[email protected]>
AuthorDate: Thu Jun 25 19:58:24 2026 -0700
[MINOR] Guard detailed metadata size metrics with a config (#18803)
* feat(metadata): guard detailed metadata size metrics behind a config
Detailed size metrics in the metadata table cause FSV to be built for MDT
in the driver. Add hoodie.metadata.enable.detailed.metrics (default false)
to gate emission of these detailed size metrics, reducing driver memory
pressure at scale.
Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
---------
Co-authored-by: Claude Opus 4.7 (1M context) <[email protected]>
---
.../metadata/HoodieBackedTableMetadataWriter.java | 6 +++++-
.../table/action/index/RunIndexActionExecutor.java | 2 +-
.../FlinkHoodieBackedTableMetadataWriter.java | 2 +-
.../JavaHoodieBackedTableMetadataWriter.java | 2 +-
.../hudi/testutils/TestHoodieMetadataBase.java | 1 +
.../SparkHoodieBackedTableMetadataWriter.java | 2 +-
...ieBackedTableMetadataWriterTableVersionSix.java | 2 +-
.../client/functional/TestHoodieMetadataBase.java | 1 +
.../hudi/common/config/HoodieMetadataConfig.java | 18 ++++++++++++++++
.../apache/hudi/metadata/BaseTableMetadata.java | 3 ++-
.../hudi/metadata/HoodieMetadataMetrics.java | 8 +++++++-
.../common/config/TestHoodieMetadataConfig.java | 24 ++++++++++++++++++++++
12 files changed, 63 insertions(+), 8 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index 7e75b71b14f4..0fb4136ea5c6 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -1984,7 +1984,11 @@ public abstract class HoodieBackedTableMetadataWriter<I,
O> implements HoodieTab
metadataMetaClient.reloadActiveTimeline();
// Update total size of the metadata and count of base/log files
- metrics.ifPresent(m -> m.updateSizeMetrics(metadataMetaClient, metadata,
dataMetaClient.getTableConfig().getMetadataPartitions()));
+ metrics.ifPresent(m -> {
+ if (m.isDetailedMetricsEnabled()) {
+ m.updateSizeMetrics(metadataMetaClient, metadata,
dataMetaClient.getTableConfig().getMetadataPartitions());
+ }
+ });
}
protected abstract void bulkInsertAndCommit(BaseHoodieWriteClient<?, I, ?,
O> writeClient, String instantTime, I preppedRecordInputs,
Option<BulkInsertPartitioner> bulkInsertPartitioner);
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java
index 6bb15573c829..c0a29695bf45 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/index/RunIndexActionExecutor.java
@@ -98,7 +98,7 @@ public class RunIndexActionExecutor<T, I, K, O> extends
BaseActionExecutor<T, I,
super(context, config, table, instantTime);
this.txnManager = new TransactionManager(config, table.getStorage());
if (config.getMetadataConfig().isMetricsEnabled()) {
- this.metrics = Option.of(new
HoodieMetadataMetrics(config.getMetricsConfig(), table.getStorage()));
+ this.metrics = Option.of(new
HoodieMetadataMetrics(config.getMetricsConfig(), table.getStorage(),
config.getMetadataConfig().isDetailedMetricsEnabled()));
} else {
this.metrics = Option.empty();
}
diff --git
a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java
b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java
index 268ae942b92e..8cc44f068df5 100644
---
a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java
+++
b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java
@@ -98,7 +98,7 @@ public class FlinkHoodieBackedTableMetadataWriter extends
HoodieBackedTableMetad
protected void initRegistry() {
if (metadataWriteConfig.isMetricsOn()) {
// should support executor metrics
- this.metrics = Option.of(new
HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(),
dataMetaClient.getStorage()));
+ this.metrics = Option.of(new
HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(),
dataMetaClient.getStorage(),
dataWriteConfig.getMetadataConfig().isDetailedMetricsEnabled()));
} else {
this.metrics = Option.empty();
}
diff --git
a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java
b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java
index 2a035f560d5c..b5d9d2822f19 100644
---
a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java
+++
b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java
@@ -82,7 +82,7 @@ public class JavaHoodieBackedTableMetadataWriter extends
HoodieBackedTableMetada
@Override
protected void initRegistry() {
if (metadataWriteConfig.isMetricsOn()) {
- this.metrics = Option.of(new
HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(),
dataMetaClient.getStorage()));
+ this.metrics = Option.of(new
HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(),
dataMetaClient.getStorage(),
dataWriteConfig.getMetadataConfig().isDetailedMetricsEnabled()));
} else {
this.metrics = Option.empty();
}
diff --git
a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java
b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java
index b555b7696461..792f742ed92a 100644
---
a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java
+++
b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java
@@ -308,6 +308,7 @@ public class TestHoodieMetadataBase extends
HoodieJavaClientTestHarness {
.enable(useFileListingMetadata)
.withMetadataIndexColumnStats(false)
.enableMetrics(enableMetrics)
+ .enableDetailedMetadataMetrics(enableMetrics)
.ignoreSpuriousDeletes(validateMetadataPayloadConsistency)
.withMetadataIndexColumnStats(false) // HUDI-8774
.withEngineType(EngineType.JAVA)
diff --git
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java
index dfb295d5d16a..7ea05926768d 100644
---
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java
+++
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java
@@ -138,7 +138,7 @@ public class SparkHoodieBackedTableMetadataWriter extends
HoodieBackedTableMetad
} else {
registry = Registry.getRegistry("HoodieMetadata");
}
- this.metrics = Option.of(new
HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(),
dataMetaClient.getStorage()));
+ this.metrics = Option.of(new
HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(),
dataMetaClient.getStorage(),
dataWriteConfig.getMetadataConfig().isDetailedMetricsEnabled()));
} else {
this.metrics = Option.empty();
}
diff --git
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java
index 617b4b4e820f..facd02965749 100644
---
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java
+++
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java
@@ -106,7 +106,7 @@ public class
SparkHoodieBackedTableMetadataWriterTableVersionSix extends HoodieB
} else {
registry = Registry.getRegistry("HoodieMetadata");
}
- this.metrics = Option.of(new
HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(),
dataMetaClient.getStorage()));
+ this.metrics = Option.of(new
HoodieMetadataMetrics(metadataWriteConfig.getMetricsConfig(),
dataMetaClient.getStorage(),
dataWriteConfig.getMetadataConfig().isDetailedMetricsEnabled()));
} else {
this.metrics = Option.empty();
}
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java
index f28de3266c7f..4a210227bb74 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBase.java
@@ -344,6 +344,7 @@ public class TestHoodieMetadataBase extends
HoodieSparkClientTestHarness {
.withMetadataConfig(HoodieMetadataConfig.newBuilder()
.enable(useFileListingMetadata)
.enableMetrics(enableMetrics)
+ .enableDetailedMetadataMetrics(enableMetrics)
.ignoreSpuriousDeletes(validateMetadataPayloadConsistency)
.build())
.withMetricsConfig(HoodieMetricsConfig.newBuilder().on(enableMetrics)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
index cd4d03ee4ee7..7ce1e447e65f 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
@@ -683,6 +683,15 @@ public final class HoodieMetadataConfig extends
HoodieConfig {
+ "with the actual record count stored in the metadata table. This
validation runs in a distributed manner "
+ "using the compute engine. Disabled by default as it adds overhead
to the initialization process.");
+ public static final ConfigProperty<Boolean> ENABLE_DETAILED_METRICS =
ConfigProperty
+ .key(METADATA_PREFIX + ".enable.detailed.metrics")
+ .defaultValue(false)
+ .markAdvanced()
+ .sinceVersion("1.3.0")
+ .withDocumentation("Enables detailed metadata table metrics —
per-metadata-partition file size and base/log "
+ + "file counts. Emitting these requires building a
HoodieTableFileSystemView for the metadata table on "
+ + "the driver, which adds memory pressure at scale; leave disabled
unless you need the breakdown.");
+
public long getMaxLogFileSize() {
return getLong(MAX_LOG_FILE_SIZE_BYTES_PROP);
}
@@ -1020,6 +1029,10 @@ public final class HoodieMetadataConfig extends
HoodieConfig {
return subIndexNameToDrop.contains(indexName);
}
+ public boolean isDetailedMetricsEnabled() {
+ return getBoolean(ENABLE_DETAILED_METRICS);
+ }
+
public static class Builder {
private EngineType engineType = EngineType.SPARK;
@@ -1349,6 +1362,11 @@ public final class HoodieMetadataConfig extends
HoodieConfig {
return this;
}
+ public Builder enableDetailedMetadataMetrics(boolean enable) {
+ metadataConfig.setValue(ENABLE_DETAILED_METRICS, String.valueOf(enable));
+ return this;
+ }
+
public HoodieMetadataConfig build() {
metadataConfig.setDefaultValue(ENABLE,
getDefaultMetadataEnable(engineType));
metadataConfig.setDefaultValue(ENABLE_METADATA_INDEX_COLUMN_STATS,
getDefaultColStatsEnable(engineType));
diff --git
a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
index bf6796e62e34..31c6b55511dc 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
@@ -94,7 +94,8 @@ public abstract class BaseTableMetadata extends
AbstractHoodieTableMetadata {
if (metadataConfig.isMetricsEnabled()) {
this.metrics = Option.of(new
HoodieMetadataMetrics(HoodieMetricsConfig.newBuilder()
-
.fromProperties(metadataConfig.getProps()).withPath(dataBasePath).build(),
dataMetaClient.getStorage()));
+
.fromProperties(metadataConfig.getProps()).withPath(dataBasePath).build(),
dataMetaClient.getStorage(),
+ metadataConfig.isDetailedMetricsEnabled()));
} else {
this.metrics = Option.empty();
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
index c7244a30d094..32f7eca343f3 100644
---
a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
+++
b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
@@ -85,10 +85,12 @@ public class HoodieMetadataMetrics implements Serializable {
private final transient MetricRegistry metricsRegistry;
private final transient Metrics metrics;
+ private final boolean detailedMetricsEnabled;
- public HoodieMetadataMetrics(HoodieMetricsConfig metricsConfig,
HoodieStorage storage) {
+ public HoodieMetadataMetrics(HoodieMetricsConfig metricsConfig,
HoodieStorage storage, boolean detailedMetricsEnabled) {
this.metrics = Metrics.getInstance(metricsConfig, storage);
this.metricsRegistry = metrics.getRegistry();
+ this.detailedMetricsEnabled = detailedMetricsEnabled;
}
public Map<String, String> getStats(boolean detailed, HoodieTableMetaClient
metaClient, HoodieTableMetadata metadata, Set<String> metadataPartitions) {
@@ -101,6 +103,10 @@ public class HoodieMetadataMetrics implements Serializable
{
}
}
+ public boolean isDetailedMetricsEnabled() {
+ return detailedMetricsEnabled;
+ }
+
private Map<String, String> getStats(HoodieTableFileSystemView fsView,
boolean detailed, HoodieTableMetadata tableMetadata, Set<String>
metadataPartitions)
throws IOException {
Map<String, String> stats = new HashMap<>();
diff --git
a/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieMetadataConfig.java
b/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieMetadataConfig.java
index b66b432abc24..f3091ed58fe3 100644
---
a/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieMetadataConfig.java
+++
b/hudi-common/src/test/java/org/apache/hudi/common/config/TestHoodieMetadataConfig.java
@@ -269,4 +269,28 @@ class TestHoodieMetadataConfig {
.withTableServiceManagerEnabled(true)
.build());
}
+
+ @Test
+ void testMetricsConfig() {
+ // Test default value
+ HoodieMetadataConfig config = HoodieMetadataConfig.newBuilder().build();
+ assertFalse(config.isMetricsEnabled());
+
+ Properties props = new Properties();
+ props.put(HoodieMetadataConfig.METRICS_ENABLE.key(), true);
+ config = HoodieMetadataConfig.newBuilder()
+ .fromProperties(props)
+ .build();
+ assertTrue(config.isMetricsEnabled());
+ assertFalse(config.isDetailedMetricsEnabled());
+
+ props = new Properties();
+ props.put(HoodieMetadataConfig.METRICS_ENABLE.key(), true);
+ props.put(HoodieMetadataConfig.ENABLE_DETAILED_METRICS.key(), true);
+ config = HoodieMetadataConfig.newBuilder()
+ .fromProperties(props)
+ .build();
+ assertTrue(config.isMetricsEnabled());
+ assertTrue(config.isDetailedMetricsEnabled());
+ }
}
