rahil-c commented on code in PR #19575: URL: https://github.com/apache/hudi/pull/19575#discussion_r3859019676
########## hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/ExecutorMetricRegistry.java: ########## @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.metrics; + +import org.apache.hudi.common.metrics.Registry; +import org.apache.hudi.config.HoodieWriteConfig; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.function.Predicate; + +/** + * Every class of executor-collected metric, and the only thing a new one is added to. The driver must + * declare it up front because an {@code AccumulatorV2} must be registered with the {@code SparkContext} + * before a task can contribute; the bundle sent to executors and the commit drain both iterate this. + */ +public enum ExecutorMetricRegistry implements ExecutorMetricGroup { + + RECORD_INDEX_LOOKUP( + "HoodieRecordIndexLookup", + "hoodie.rli.lookup.", + "rli", + "lookup", + HoodieWriteConfig::isRecordIndexLookupMetricsEnabled); + + private final String registryName; + private final String commitMetadataPrefix; + private final String metricAction; + private final String metricQualifier; + private final Predicate<HoodieWriteConfig> enabled; + + ExecutorMetricRegistry(String registryName, String commitMetadataPrefix, String metricAction, + String metricQualifier, Predicate<HoodieWriteConfig> enabled) { + this.registryName = registryName; + this.commitMetadataPrefix = commitMetadataPrefix; + this.metricAction = metricAction; + this.metricQualifier = metricQualifier; + this.enabled = enabled; + } + + /** The bare name emitting code passes to {@link Registry#getRegistry(String)}. */ + @Override + public String registryName() { + return registryName; + } + + @Override + public String commitMetadataPrefix() { + return commitMetadataPrefix; + } + + @Override + public String metricAction() { + return metricAction; + } + + @Override + public String metricQualifier() { + return metricQualifier; + } + + /** Gating here, rather than in the drain, is what lets a new class of metric need no new config. */ + @Override + public boolean isEnabled(HoodieWriteConfig config) { + return enabled.test(config); + } + + /** + * Driver-side {@code REGISTRY_MAP} key. Table name alone is not an identity: two tables can share one + * and would then share a registry. Executors use the bare {@link #registryName()}. + */ + @Override + public String scopedName(String basePath) { Review Comment: Real, and I hit it independently from the other direction while debugging: `Metrics.shutdown()` running `Registry.getAllMetrics(true, true)` is also why an abandoned commit's counters cannot survive to the retry on the DataSource path. In practice the republish does not fire today, because release-by-subtraction leaves the registry empty by the time teardown runs -- so it needs leftovers to be visible at all. Deferring the digest-free name / scrape exclusion to the registry-lifetime follow-up, since the digest only exists to disambiguate inside the process-wide map and both go away together. ########## hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/ExecutorMetricRegistry.java: ########## @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.metrics; + +import org.apache.hudi.common.metrics.Registry; +import org.apache.hudi.config.HoodieWriteConfig; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.function.Predicate; + +/** + * Every class of executor-collected metric, and the only thing a new one is added to. The driver must + * declare it up front because an {@code AccumulatorV2} must be registered with the {@code SparkContext} + * before a task can contribute; the bundle sent to executors and the commit drain both iterate this. + */ +public enum ExecutorMetricRegistry implements ExecutorMetricGroup { + + RECORD_INDEX_LOOKUP( + "HoodieRecordIndexLookup", + "hoodie.rli.lookup.", + "rli", + "lookup", + HoodieWriteConfig::isRecordIndexLookupMetricsEnabled); + + private final String registryName; + private final String commitMetadataPrefix; + private final String metricAction; + private final String metricQualifier; + private final Predicate<HoodieWriteConfig> enabled; + + ExecutorMetricRegistry(String registryName, String commitMetadataPrefix, String metricAction, + String metricQualifier, Predicate<HoodieWriteConfig> enabled) { + this.registryName = registryName; + this.commitMetadataPrefix = commitMetadataPrefix; + this.metricAction = metricAction; + this.metricQualifier = metricQualifier; + this.enabled = enabled; + } + + /** The bare name emitting code passes to {@link Registry#getRegistry(String)}. */ + @Override + public String registryName() { + return registryName; + } + + @Override + public String commitMetadataPrefix() { + return commitMetadataPrefix; + } + + @Override + public String metricAction() { + return metricAction; + } + + @Override + public String metricQualifier() { + return metricQualifier; + } + + /** Gating here, rather than in the drain, is what lets a new class of metric need no new config. */ + @Override + public boolean isEnabled(HoodieWriteConfig config) { + return enabled.test(config); + } + + /** + * Driver-side {@code REGISTRY_MAP} key. Table name alone is not an identity: two tables can share one + * and would then share a registry. Executors use the bare {@link #registryName()}. + */ + @Override + public String scopedName(String basePath) { Review Comment: Related to this, and worth recording: the digest in this name was not only ugly, it was wrong. Digesting the raw base path meant one table hashed two ways -- scheme-qualified from the Spark SQL catalog location, bare from the DataSource -- so that write path silently reported nothing at all. Now fixed by digesting authority + path. That does not change your point. The digest exists only because the map is process-wide, and both it and the common-scrape republish go away together when the registry becomes instance-scoped. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
