danny0405 commented on code in PR #19575: URL: https://github.com/apache/hudi/pull/19575#discussion_r3877796191
########## hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metrics/RecordIndexLookupMetrics.java: ########## @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.metrics; + +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.metrics.Registry; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.storage.StoragePath; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * The record index lookup counters, end to end: what they are called, where they are collected, and how + * they reach the reporter once a commit lands. + * + * <p>The numbers are only knowable on executors, because a lookup returns its hits and a miss produces + * no output row at all. So the driver resolves an accumulator-backed {@link DistributedRegistry}, the + * lookup closure captures it, tasks add into their own copy, and Spark merges those copies home. + */ +public class RecordIndexLookupMetrics { + + private static final Logger LOG = LoggerFactory.getLogger(RecordIndexLookupMetrics.class); + + /** The registry name. Scoping for the driver-side key is applied by {@link #scopedName}. */ + public static final String REGISTRY_NAME = "HoodieRecordIndexLookup"; + + /** Reporter naming, as passed to {@code HoodieMetrics.getMetricsName}. */ + public static final String METRIC_ACTION = "rli"; + public static final String METRIC_QUALIFIER = "lookup"; + + /** Counts records, not distinct keys: a batch repeating a key contributes once per record, which is + * what keeps {@code hits + misses == key_count} exact. */ + public static final String KEY_COUNT = "lookup_record_index_key_count"; + public static final String KEY_HIT_COUNT = "lookup_record_index_key_hit_count"; + public static final String KEY_MISS_COUNT = "lookup_record_index_key_miss_count"; + public static final String SHARDS_READ = "lookup_record_index_shards_read"; + /** + * Wall-clock spent in the shard read, summed across shards rather than averaged because shards are read + * in parallel: the value is per-commit read effort, and dividing by {@link #SHARDS_READ} gives a mean. + * + * <p>Distinct from {@code index.lookup.duration} published by {@code HoodieMetrics.updateIndexMetrics}, + * which is driver wall-clock for the whole {@code tagLocation} including scheduling. Comparing the two + * shows how much of a lookup was actually spent reading the index. + */ + public static final String LOOKUP_TIME = "lookup_record_index_time"; + + private RecordIndexLookupMetrics() { + } + + /** + * The registry a lookup task collects into, or null when nothing should be collected. Captured in the + * lookup closure and passed to {@link #recordShardLookup}, so delivery is by closure capture rather + * than by name. + * + * <p>Requires the reporter to be on as well: with {@code hoodie.metrics.on} off there is nowhere to + * publish, and collecting would register an accumulator and scan every shard for nothing. + */ + public static Registry resolveRegistry(HoodieEngineContext context, HoodieWriteConfig config) { Review Comment: make it return Option instead. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
