This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new dbc6ac50aec [HUDI-7395] Fix computation for metrics in
HoodieMetadataMetrics (#10641)
dbc6ac50aec is described below
commit dbc6ac50aec2f9724c46d67c0ab4a3f9b2489b75
Author: Lokesh Jain <[email protected]>
AuthorDate: Mon Jun 24 04:40:16 2024 +0530
[HUDI-7395] Fix computation for metrics in HoodieMetadataMetrics (#10641)
---
.../apache/hudi/metadata/HoodieBackedTableMetadataWriter.java | 8 ++++----
.../java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java | 5 +++++
.../main/java/org/apache/hudi/metadata/BaseTableMetadata.java | 9 +++++----
.../java/org/apache/hudi/metadata/HoodieMetadataMetrics.java | 9 ++++++---
4 files changed, 20 insertions(+), 11 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index 8dc738b59ed..89d21e79b22 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -299,7 +299,7 @@ public abstract class HoodieBackedTableMetadataWriter<I>
implements HoodieTableM
}
if (reInitialize) {
- metrics.ifPresent(m ->
m.updateMetrics(HoodieMetadataMetrics.REBOOTSTRAP_STR, 1));
+ metrics.ifPresent(m ->
m.incrementMetric(HoodieMetadataMetrics.REBOOTSTRAP_STR, 1));
LOG.info("Deleting Metadata Table directory so that it can be
re-initialized");
HoodieTableMetadataUtil.deleteMetadataTable(dataMetaClient,
engineContext, false);
exists = false;
@@ -1484,11 +1484,11 @@ public abstract class
HoodieBackedTableMetadataWriter<I> implements HoodieTableM
throw e;
} finally {
long timeSpent = metadataTableServicesTimer.endTimer();
- metrics.ifPresent(m ->
m.updateMetrics(HoodieMetadataMetrics.TABLE_SERVICE_EXECUTION_DURATION,
timeSpent));
+ metrics.ifPresent(m ->
m.setMetric(HoodieMetadataMetrics.TABLE_SERVICE_EXECUTION_DURATION, timeSpent));
if (allTableServicesExecutedSuccessfullyOrSkipped) {
- metrics.ifPresent(m ->
m.incrementMetric(HoodieMetadataMetrics.TABLE_SERVICE_EXECUTION_STATUS, 1));
+ metrics.ifPresent(m ->
m.setMetric(HoodieMetadataMetrics.TABLE_SERVICE_EXECUTION_STATUS, 1));
} else {
- metrics.ifPresent(m ->
m.incrementMetric(HoodieMetadataMetrics.TABLE_SERVICE_EXECUTION_STATUS, -1));
+ metrics.ifPresent(m ->
m.setMetric(HoodieMetadataMetrics.TABLE_SERVICE_EXECUTION_STATUS, -1));
}
}
}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java
index bd4c82a2d84..3caff83eb05 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java
@@ -48,6 +48,7 @@ import java.util.Properties;
import static
org.apache.hudi.common.config.HoodieMetadataConfig.DEFAULT_METADATA_ASYNC_CLEAN;
import static
org.apache.hudi.common.config.HoodieMetadataConfig.DEFAULT_METADATA_CLEANER_COMMITS_RETAINED;
import static
org.apache.hudi.common.config.HoodieMetadataConfig.DEFAULT_METADATA_POPULATE_META_FIELDS;
+import static org.apache.hudi.common.util.StringUtils.nonEmpty;
import static
org.apache.hudi.metadata.HoodieTableMetadata.METADATA_TABLE_NAME_SUFFIX;
/**
@@ -160,6 +161,10 @@ public class HoodieMetadataWriteUtils {
final Properties properties = new Properties();
properties.put(HoodieTableConfig.RECORDKEY_FIELDS.key(),
RECORD_KEY_FIELD_NAME);
properties.put("hoodie.datasource.write.recordkey.field",
RECORD_KEY_FIELD_NAME);
+ if (nonEmpty(writeConfig.getMetricReporterMetricsNamePrefix())) {
+ properties.put(HoodieMetricsConfig.METRICS_REPORTER_PREFIX.key(),
+ writeConfig.getMetricReporterMetricsNamePrefix() +
METADATA_TABLE_NAME_SUFFIX);
+ }
builder.withProperties(properties);
if (writeConfig.isMetricsOn()) {
diff --git
a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
index 633ab7cd1da..bc858a2a15e 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
@@ -207,8 +207,8 @@ public abstract class BaseTableMetadata extends
AbstractHoodieTableMetadata {
List<String> partitionIDFileIDStringsList = new
ArrayList<>(partitionIDFileIDStrings);
Map<String, HoodieRecord<HoodieMetadataPayload>> hoodieRecords =
getRecordsByKeys(partitionIDFileIDStringsList,
MetadataPartitionType.BLOOM_FILTERS.getPartitionPath());
- metrics.ifPresent(m ->
m.updateMetrics(HoodieMetadataMetrics.LOOKUP_BLOOM_FILTERS_METADATA_STR,
- (timer.endTimer() / partitionIDFileIDStrings.size())));
+ metrics.ifPresent(m ->
m.updateMetrics(HoodieMetadataMetrics.LOOKUP_BLOOM_FILTERS_METADATA_STR,
timer.endTimer()));
+ metrics.ifPresent(m ->
m.setMetric(HoodieMetadataMetrics.LOOKUP_BLOOM_FILTERS_FILE_COUNT_STR,
partitionIDFileIDStringsList.size()));
Map<Pair<String, String>, BloomFilter> partitionFileToBloomFilterMap = new
HashMap<>(hoodieRecords.size());
for (final Map.Entry<String, HoodieRecord<HoodieMetadataPayload>> entry :
hoodieRecords.entrySet()) {
@@ -259,6 +259,7 @@ public abstract class BaseTableMetadata extends
AbstractHoodieTableMetadata {
Map<String, HoodieRecord<HoodieMetadataPayload>> hoodieRecords =
getRecordsByKeys(columnStatKeylist,
MetadataPartitionType.COLUMN_STATS.getPartitionPath());
metrics.ifPresent(m ->
m.updateMetrics(HoodieMetadataMetrics.LOOKUP_COLUMN_STATS_METADATA_STR,
timer.endTimer()));
+ metrics.ifPresent(m ->
m.setMetric(HoodieMetadataMetrics.LOOKUP_COLUMN_STATS_FILE_COUNT_STR,
columnStatKeylist.size()));
Map<Pair<String, String>, HoodieMetadataColumnStats> fileToColumnStatMap =
new HashMap<>();
for (final Map.Entry<String, HoodieRecord<HoodieMetadataPayload>> entry :
hoodieRecords.entrySet()) {
@@ -306,8 +307,8 @@ public abstract class BaseTableMetadata extends
AbstractHoodieTableMetadata {
}));
metrics.ifPresent(m ->
m.updateMetrics(HoodieMetadataMetrics.LOOKUP_RECORD_INDEX_TIME_STR,
timer.endTimer()));
- metrics.ifPresent(m ->
m.updateMetrics(HoodieMetadataMetrics.LOOKUP_RECORD_INDEX_KEYS_COUNT_STR,
recordKeys.size()));
- metrics.ifPresent(m ->
m.updateMetrics(HoodieMetadataMetrics.LOOKUP_RECORD_INDEX_KEYS_HITS_COUNT_STR,
recordKeyToLocation.size()));
+ metrics.ifPresent(m ->
m.setMetric(HoodieMetadataMetrics.LOOKUP_RECORD_INDEX_KEYS_COUNT_STR,
recordKeys.size()));
+ metrics.ifPresent(m ->
m.setMetric(HoodieMetadataMetrics.LOOKUP_RECORD_INDEX_KEYS_HITS_COUNT_STR,
recordKeyToLocation.size()));
return recordKeyToLocation;
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
index 25a79cb12d8..e7392a8f1ac 100644
---
a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
+++
b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataMetrics.java
@@ -52,16 +52,19 @@ public class HoodieMetadataMetrics implements Serializable {
public static final String LOOKUP_FILES_STR = "lookup_files";
public static final String LOOKUP_BLOOM_FILTERS_METADATA_STR =
"lookup_meta_index_bloom_filters";
public static final String LOOKUP_COLUMN_STATS_METADATA_STR =
"lookup_meta_index_column_ranges";
+ public static final String LOOKUP_BLOOM_FILTERS_FILE_COUNT_STR =
"lookup_meta_index_bloom_filters_file_count";
+ public static final String LOOKUP_COLUMN_STATS_FILE_COUNT_STR =
"lookup_meta_index_column_ranges_file_count";
+
// Time for lookup from record index
public static final String LOOKUP_RECORD_INDEX_TIME_STR =
"lookup_record_index_time";
// Number of keys looked up in a call
public static final String LOOKUP_RECORD_INDEX_KEYS_COUNT_STR =
"lookup_record_index_key_count";
// Number of keys found in record index
- public static final String LOOKUP_RECORD_INDEX_KEYS_HITS_COUNT_STR =
"lookup_record_index_key_count";
+ public static final String LOOKUP_RECORD_INDEX_KEYS_HITS_COUNT_STR =
"lookup_record_index_key_hit_count";
public static final String SCAN_STR = "scan";
public static final String BASEFILE_READ_STR = "basefile_read";
public static final String INITIALIZE_STR = "initialize";
- public static final String REBOOTSTRAP_STR = "rebootstrap";
+ public static final String REBOOTSTRAP_STR = "rebootstrap_count";
public static final String BOOTSTRAP_ERR_STR = "bootstrap_error";
// Stats names
@@ -143,7 +146,7 @@ public class HoodieMetadataMetrics implements Serializable {
String countKey = action + ".count";
String durationKey = action + ".totalDuration";
incrementMetric(countKey, 1);
- incrementMetric(durationKey, durationInMs);
+ setMetric(durationKey, durationInMs);
}
public void updateSizeMetrics(HoodieTableMetaClient metaClient,
HoodieBackedTableMetadata metadata, Set<String> metadataPartitions) {
