hudi-agent commented on code in PR #20153: URL: https://github.com/apache/hudi/pull/20153#discussion_r4140808344
########## hudi-utilities/src/main/java/org/apache/hudi/utilities/health/RecordIndexSizingCheck.java: ########## @@ -0,0 +1,420 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.utilities.health; + +import org.apache.hudi.common.config.ConfigProperty; +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.engine.HoodieLocalEngineContext; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.ConfigUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.metadata.HoodieBackedTableMetadata; +import org.apache.hudi.metadata.HoodieTableMetadataUtil; +import org.apache.hudi.metadata.MetadataPartitionType; + +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static org.apache.hudi.metadata.HoodieTableMetadataUtil.RECORD_INDEX_AVERAGE_RECORD_SIZE; + +/** + * Checks whether the record-level index is sized appropriately for the table it serves. + * + * <p>The number of file groups in the record index is decided once, when the index is initialized, + * from the record count at that moment and the writer's sizing configuration. It never changes + * afterwards: every subsequent index write lands in one of those same file groups. A table that has + * grown well past what its index was sized for therefore has file groups that are each far larger + * than the writer's configuration intended, which slows every index lookup and concentrates every + * index write on the same few file groups. + * + * <p>So the check re-runs the writer's own sizing estimate against the table as it is now and + * compares that ideal against the file-group count the index actually has. It also looks at the + * largest file group directly, since a group past the configured maximum size is the symptom the + * operator will feel regardless of how the count came out. Both signals share one remedy: the index + * has to be re-bootstrapped with corrected sizing, which is disruptive and is spelled out in the + * findings. + * + * <p>A partitioned record index is sized per data partition, not as a whole, so for that layout + * the count comparison is made per data partition and the worst one is reported. Which layout the + * writer sizes for is read from its configuration and which layout is on storage is read from the + * index's file-group names; if the two disagree, the count comparison is skipped and the report says + * so, while the file-group size rule still runs. + * + * <p>The record count is estimated from the index's own footprint on storage divided by the same + * average record size the writer sizes with, so that no data file is scanned. The estimate is + * conservative -- index files are compressed, so the true count is higher -- and it is labelled as + * an estimate in the report. + */ +public class RecordIndexSizingCheck implements HealthCheck { + + static final String NAME = "record-index"; + + /** + * Actual-to-ideal file-group ratio below which the index is called undersized. The ideal count + * already carries the writer's growth factor, so an index initialized on a smaller table sits + * somewhat below 1.0 during ordinary growth. The slack keeps that from alerting while a table + * that has grown past what its index was sized for still does. + */ + static final double MIN_ACCEPTABLE_FILE_GROUP_RATIO = 0.5; + + /** + * How far past the configured maximum a single file group may grow before the index is called + * unhealthy. File groups are bounded only at initialization; between metadata compactions, log + * files push a group past the configured size transiently, so alerting at exactly the maximum + * would fire on healthy tables. + */ + static final double MAX_FILE_GROUP_SIZE_SLACK_FACTOR = 1.5; + + @Override + public String getName() { + return NAME; + } + + @Override + public String getDescription() { + return "Whether the record-level index has enough file groups for the table it serves"; + } + + @Override + public boolean appliesTo(HealthCheckContext context) { + return context.getMetaClient().getTableConfig().isMetadataPartitionAvailable(MetadataPartitionType.RECORD_INDEX); + } + + @Override + public HealthCheckResult check(HealthCheckContext context) throws Exception { + if (!appliesTo(context)) { + return HealthCheckResult.skipped(NAME, String.format( + "Table has no record-level index: '%s' in the table configuration does not list '%s'", + HoodieTableConfig.TABLE_METADATA_PARTITIONS.key(), MetadataPartitionType.RECORD_INDEX.getPartitionPath())); + } + + TypedProperties props = context.getProps(); + boolean partitionedIndex = ConfigUtils.getBooleanWithAltKeys(props, HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP); + ConfigProperty<Integer> minCountProp = partitionedIndex + ? HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP + : HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP; + ConfigProperty<Integer> maxCountProp = partitionedIndex + ? HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP + : HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP; + + Option<String> missing = firstMissingConfig(context, minCountProp, maxCountProp, + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP, + HoodieMetadataConfig.RECORD_INDEX_GROWTH_FACTOR_PROP); + if (missing.isPresent()) { + return context.skipForMissingConfig(NAME, missing.get()); + } + + SizingConfig sizing = new SizingConfig( + Integer.parseInt(configValue(props, minCountProp)), + Integer.parseInt(configValue(props, maxCountProp)), + Float.parseFloat(configValue(props, HoodieMetadataConfig.RECORD_INDEX_GROWTH_FACTOR_PROP)), + Long.parseLong(configValue(props, HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP))); + + HealthCheckResult.Builder result = HealthCheckResult.builder(NAME) + .config(HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key(), partitionedIndex) + .config(minCountProp.key(), sizing.minFileGroupCount) + .config(maxCountProp.key(), sizing.maxFileGroupCount) + .config(HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key(), sizing.maxFileGroupSizeBytes) + .config(HoodieMetadataConfig.RECORD_INDEX_GROWTH_FACTOR_PROP.key(), sizing.growthFactor) + .config("effective.average.record.size.bytes", RECORD_INDEX_AVERAGE_RECORD_SIZE); + + if (sizing.maxFileGroupSizeBytes < RECORD_INDEX_AVERAGE_RECORD_SIZE) { + return result.status(HealthStatus.SKIPPED) + .summary(String.format( + "'%s' is %d bytes, smaller than the %d-byte average index record the writer sizes with, so no " + + "file-group count can be estimated. The writer cannot size an index with this value either.", + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key(), sizing.maxFileGroupSizeBytes, + RECORD_INDEX_AVERAGE_RECORD_SIZE)) + .build(); + } + + RecordIndexFootprint footprint = measureRecordIndex(context.getMetaClient()); + long maxHealthyFileGroupBytes = (long) (MAX_FILE_GROUP_SIZE_SLACK_FACTOR * sizing.maxFileGroupSizeBytes); + result.config("effective.max.healthy.file.group.bytes", maxHealthyFileGroupBytes) + .config("observed.file.group.count", footprint.fileGroupCount) + .config("observed.base.file.count", footprint.baseFileCount) + .config("observed.log.file.count", footprint.logFileCount) + .config("observed.total.size.bytes", footprint.totalSizeBytes) + .config("observed.largest.file.group.bytes", footprint.largestFileGroupBytes) + .config("observed.largest.file.group.id", footprint.largestFileGroupId); + + Option<CountComparison> comparison = compareFileGroupCount(footprint, partitionedIndex, sizing, result); + boolean undersized = comparison.isPresent() && comparison.get().isUndersized(); + boolean oversizedFileGroup = footprint.largestFileGroupBytes > maxHealthyFileGroupBytes; + + if (!undersized && !oversizedFileGroup) { + return result.status(HealthStatus.HEALTHY) + .summary(comparison.isPresent() + ? String.format("%s has %d file group(s) against an ideal of %d for an estimated %d records; largest " + + "file group is %d bytes", comparison.get().subject(), comparison.get().observedFileGroupCount, + comparison.get().idealFileGroupCount, comparison.get().estimatedRecordCount, footprint.largestFileGroupBytes) + : String.format("Record index has %d file group(s); largest file group is %d bytes, within the " + + "configured maximum. File-group count was not compared against the writer's sizing.", + footprint.fileGroupCount, footprint.largestFileGroupBytes)) + .build(); + } + + result.status(HealthStatus.UNHEALTHY); + if (undersized) { + CountComparison shortfall = comparison.get(); + result.summary(String.format( + "%s has %d file group(s) but the writer's sizing calls for %d (at least %d to be healthy)", + shortfall.subject(), shortfall.observedFileGroupCount, shortfall.idealFileGroupCount, + shortfall.minHealthyFileGroupCount())) + .finding(String.format( + "%s was initialized with %d file group(s). Its %d bytes on storage put it at an estimated %d records " Review Comment: 🤖 When min == max, or the estimate is clamped up to `min`, the ideal count ignores table size. A tiny index then gets UNHEALTHY, a note that each file group "holds roughly 5.0x more keys than intended", and a maintenance-window re-bootstrap (see `applyAllDefaultsEvaluatesWithoutSuppliedConfig`). Could we detect when the ideal comes from the min floor and word or grade that case differently? <sub><i>⚠️ AI-generated; verify before applying. React 👍/👎 to flag quality.</i></sub> ########## hudi-utilities/src/main/java/org/apache/hudi/utilities/health/RecordIndexSizingCheck.java: ########## @@ -0,0 +1,420 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.utilities.health; + +import org.apache.hudi.common.config.ConfigProperty; +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.engine.HoodieLocalEngineContext; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.ConfigUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.metadata.HoodieBackedTableMetadata; +import org.apache.hudi.metadata.HoodieTableMetadataUtil; +import org.apache.hudi.metadata.MetadataPartitionType; + +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static org.apache.hudi.metadata.HoodieTableMetadataUtil.RECORD_INDEX_AVERAGE_RECORD_SIZE; + +/** + * Checks whether the record-level index is sized appropriately for the table it serves. + * + * <p>The number of file groups in the record index is decided once, when the index is initialized, + * from the record count at that moment and the writer's sizing configuration. It never changes + * afterwards: every subsequent index write lands in one of those same file groups. A table that has + * grown well past what its index was sized for therefore has file groups that are each far larger + * than the writer's configuration intended, which slows every index lookup and concentrates every + * index write on the same few file groups. + * + * <p>So the check re-runs the writer's own sizing estimate against the table as it is now and + * compares that ideal against the file-group count the index actually has. It also looks at the + * largest file group directly, since a group past the configured maximum size is the symptom the + * operator will feel regardless of how the count came out. Both signals share one remedy: the index + * has to be re-bootstrapped with corrected sizing, which is disruptive and is spelled out in the + * findings. + * + * <p>A partitioned record index is sized per data partition, not as a whole, so for that layout + * the count comparison is made per data partition and the worst one is reported. Which layout the + * writer sizes for is read from its configuration and which layout is on storage is read from the + * index's file-group names; if the two disagree, the count comparison is skipped and the report says + * so, while the file-group size rule still runs. + * + * <p>The record count is estimated from the index's own footprint on storage divided by the same + * average record size the writer sizes with, so that no data file is scanned. The estimate is + * conservative -- index files are compressed, so the true count is higher -- and it is labelled as + * an estimate in the report. + */ +public class RecordIndexSizingCheck implements HealthCheck { + + static final String NAME = "record-index"; + + /** + * Actual-to-ideal file-group ratio below which the index is called undersized. The ideal count + * already carries the writer's growth factor, so an index initialized on a smaller table sits + * somewhat below 1.0 during ordinary growth. The slack keeps that from alerting while a table + * that has grown past what its index was sized for still does. + */ + static final double MIN_ACCEPTABLE_FILE_GROUP_RATIO = 0.5; + + /** + * How far past the configured maximum a single file group may grow before the index is called + * unhealthy. File groups are bounded only at initialization; between metadata compactions, log + * files push a group past the configured size transiently, so alerting at exactly the maximum + * would fire on healthy tables. + */ + static final double MAX_FILE_GROUP_SIZE_SLACK_FACTOR = 1.5; + + @Override + public String getName() { + return NAME; + } + + @Override + public String getDescription() { + return "Whether the record-level index has enough file groups for the table it serves"; + } + + @Override + public boolean appliesTo(HealthCheckContext context) { + return context.getMetaClient().getTableConfig().isMetadataPartitionAvailable(MetadataPartitionType.RECORD_INDEX); + } + + @Override + public HealthCheckResult check(HealthCheckContext context) throws Exception { + if (!appliesTo(context)) { + return HealthCheckResult.skipped(NAME, String.format( + "Table has no record-level index: '%s' in the table configuration does not list '%s'", + HoodieTableConfig.TABLE_METADATA_PARTITIONS.key(), MetadataPartitionType.RECORD_INDEX.getPartitionPath())); + } + + TypedProperties props = context.getProps(); + boolean partitionedIndex = ConfigUtils.getBooleanWithAltKeys(props, HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP); + ConfigProperty<Integer> minCountProp = partitionedIndex + ? HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP + : HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP; + ConfigProperty<Integer> maxCountProp = partitionedIndex + ? HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP + : HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP; + + Option<String> missing = firstMissingConfig(context, minCountProp, maxCountProp, + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP, + HoodieMetadataConfig.RECORD_INDEX_GROWTH_FACTOR_PROP); + if (missing.isPresent()) { + return context.skipForMissingConfig(NAME, missing.get()); + } + + SizingConfig sizing = new SizingConfig( + Integer.parseInt(configValue(props, minCountProp)), + Integer.parseInt(configValue(props, maxCountProp)), + Float.parseFloat(configValue(props, HoodieMetadataConfig.RECORD_INDEX_GROWTH_FACTOR_PROP)), + Long.parseLong(configValue(props, HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP))); + + HealthCheckResult.Builder result = HealthCheckResult.builder(NAME) + .config(HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key(), partitionedIndex) + .config(minCountProp.key(), sizing.minFileGroupCount) + .config(maxCountProp.key(), sizing.maxFileGroupCount) + .config(HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key(), sizing.maxFileGroupSizeBytes) + .config(HoodieMetadataConfig.RECORD_INDEX_GROWTH_FACTOR_PROP.key(), sizing.growthFactor) + .config("effective.average.record.size.bytes", RECORD_INDEX_AVERAGE_RECORD_SIZE); + + if (sizing.maxFileGroupSizeBytes < RECORD_INDEX_AVERAGE_RECORD_SIZE) { + return result.status(HealthStatus.SKIPPED) + .summary(String.format( + "'%s' is %d bytes, smaller than the %d-byte average index record the writer sizes with, so no " + + "file-group count can be estimated. The writer cannot size an index with this value either.", + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key(), sizing.maxFileGroupSizeBytes, + RECORD_INDEX_AVERAGE_RECORD_SIZE)) + .build(); + } + + RecordIndexFootprint footprint = measureRecordIndex(context.getMetaClient()); + long maxHealthyFileGroupBytes = (long) (MAX_FILE_GROUP_SIZE_SLACK_FACTOR * sizing.maxFileGroupSizeBytes); + result.config("effective.max.healthy.file.group.bytes", maxHealthyFileGroupBytes) + .config("observed.file.group.count", footprint.fileGroupCount) + .config("observed.base.file.count", footprint.baseFileCount) + .config("observed.log.file.count", footprint.logFileCount) + .config("observed.total.size.bytes", footprint.totalSizeBytes) + .config("observed.largest.file.group.bytes", footprint.largestFileGroupBytes) + .config("observed.largest.file.group.id", footprint.largestFileGroupId); + + Option<CountComparison> comparison = compareFileGroupCount(footprint, partitionedIndex, sizing, result); + boolean undersized = comparison.isPresent() && comparison.get().isUndersized(); + boolean oversizedFileGroup = footprint.largestFileGroupBytes > maxHealthyFileGroupBytes; + + if (!undersized && !oversizedFileGroup) { + return result.status(HealthStatus.HEALTHY) + .summary(comparison.isPresent() + ? String.format("%s has %d file group(s) against an ideal of %d for an estimated %d records; largest " + + "file group is %d bytes", comparison.get().subject(), comparison.get().observedFileGroupCount, + comparison.get().idealFileGroupCount, comparison.get().estimatedRecordCount, footprint.largestFileGroupBytes) + : String.format("Record index has %d file group(s); largest file group is %d bytes, within the " + + "configured maximum. File-group count was not compared against the writer's sizing.", + footprint.fileGroupCount, footprint.largestFileGroupBytes)) + .build(); + } + + result.status(HealthStatus.UNHEALTHY); + if (undersized) { + CountComparison shortfall = comparison.get(); + result.summary(String.format( + "%s has %d file group(s) but the writer's sizing calls for %d (at least %d to be healthy)", + shortfall.subject(), shortfall.observedFileGroupCount, shortfall.idealFileGroupCount, + shortfall.minHealthyFileGroupCount())) + .finding(String.format( + "%s was initialized with %d file group(s). Its %d bytes on storage put it at an estimated %d records " + + "(an estimate: index bytes divided by the %d-byte average record the writer sizes with), which " + + "with the configured growth factor and maximum file-group size calls for %d. Each file group " + + "holds roughly %.1fx more keys than intended, so index lookups read larger files and every " + + "index write concentrates on the same few file groups.", + shortfall.subject(), shortfall.observedFileGroupCount, shortfall.observedSizeBytes, + shortfall.estimatedRecordCount, RECORD_INDEX_AVERAGE_RECORD_SIZE, shortfall.idealFileGroupCount, + (double) shortfall.idealFileGroupCount / Math.max(shortfall.observedFileGroupCount, 1))); + } + if (oversizedFileGroup) { + if (!undersized) { + result.summary(String.format( + "Record index file group %s is %d bytes, past the configured maximum of %d bytes", + footprint.largestFileGroupId, footprint.largestFileGroupBytes, sizing.maxFileGroupSizeBytes)); + } + result.finding(String.format( + "File group %s is %d bytes, past the configured maximum of %d bytes even with %.1fx slack. Compacting " + + "it takes proportionally longer and every lookup routed to it reads that much more. A file group " + + "that keeps growing between metadata compactions means the index has too few file groups for " + + "the table.", + footprint.largestFileGroupId, footprint.largestFileGroupBytes, sizing.maxFileGroupSizeBytes, + MAX_FILE_GROUP_SIZE_SLACK_FACTOR)); + } + result.finding(rebootstrapRemedy()); + return result.build(); + } + + /** + * Compares the file-group count against the writer's sizing estimate, at the granularity the + * writer sizes at: the whole index for a global record index, each data partition for a + * partitioned one (the worst data partition is returned). Echoes what it compared, or why it + * did not, into the result. Empty when the writer's configured layout disagrees with the layout + * on storage, since neither estimate would be comparable to the observed count then. + */ + private static Option<CountComparison> compareFileGroupCount(RecordIndexFootprint footprint, boolean partitionedIndex, + SizingConfig sizing, HealthCheckResult.Builder result) { + boolean layoutKnown = footprint.fileGroupCount > 0; + if (layoutKnown && footprint.partitionedLayout != partitionedIndex) { + result.config("effective.count.comparison", String.format( + "skipped: writer configuration says a %s record index but the index on storage is %s", + partitionedIndex ? "partitioned" : "global", footprint.partitionedLayout ? "partitioned" : "global")); + return Option.empty(); + } + + CountComparison comparison; + if (!partitionedIndex) { + result.config("effective.count.comparison", "whole index (global record index)"); + comparison = CountComparison.of(Option.empty(), footprint, sizing); + } else { + result.config("effective.count.comparison", "per data partition (partitioned record index); worst data partition reported") + .config("observed.data.partition.count", footprint.byDataPartition.size()); + comparison = CountComparison.of(Option.empty(), footprint, sizing); + for (Map.Entry<String, RecordIndexFootprint> entry : footprint.byDataPartition.entrySet()) { + CountComparison candidate = CountComparison.of(Option.of(entry.getKey()), entry.getValue(), sizing); + if (candidate.ratio() < comparison.ratio() || comparison.dataPartition.isEmpty()) { + comparison = candidate; + } + } + if (comparison.dataPartition.isPresent()) { + result.config("observed.worst.data.partition", comparison.dataPartition.get()) + .config("observed.worst.data.partition.file.group.count", comparison.observedFileGroupCount); + } + } + + result.config("effective.ideal.file.group.count", comparison.idealFileGroupCount) + .config("effective.min.healthy.file.group.count", comparison.minHealthyFileGroupCount()) + .config("observed.estimated.record.count", comparison.estimatedRecordCount); + return Option.of(comparison); + } + + /** + * The one remedy for a mis-sized record index. The file-group count is fixed at initialization, + * so there is no in-place resize; the operator has to drop the index and build it again. + */ + private static String rebootstrapRemedy() { Review Comment: 🤖 For a partitioned index, data partitions added after initialization always get `hoodie.metadata.record.level.index.min.filegroup.count` file groups (`initializeNewFileGroupsForPartitionedRLIHelper`). They never go through the size estimate. So re-bootstrapping fixes only the partitions that exist today, and the next day's partition hits the same problem. Could the partitioned remedy call out raising that min count? <sub><i>⚠️ AI-generated; verify before applying. React 👍/👎 to flag quality.</i></sub> ########## hudi-utilities/src/main/java/org/apache/hudi/utilities/health/RecordIndexSizingCheck.java: ########## @@ -0,0 +1,420 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.utilities.health; + +import org.apache.hudi.common.config.ConfigProperty; +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.engine.HoodieLocalEngineContext; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.ConfigUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.metadata.HoodieBackedTableMetadata; +import org.apache.hudi.metadata.HoodieTableMetadataUtil; +import org.apache.hudi.metadata.MetadataPartitionType; + +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static org.apache.hudi.metadata.HoodieTableMetadataUtil.RECORD_INDEX_AVERAGE_RECORD_SIZE; + +/** + * Checks whether the record-level index is sized appropriately for the table it serves. + * + * <p>The number of file groups in the record index is decided once, when the index is initialized, + * from the record count at that moment and the writer's sizing configuration. It never changes + * afterwards: every subsequent index write lands in one of those same file groups. A table that has + * grown well past what its index was sized for therefore has file groups that are each far larger + * than the writer's configuration intended, which slows every index lookup and concentrates every + * index write on the same few file groups. + * + * <p>So the check re-runs the writer's own sizing estimate against the table as it is now and + * compares that ideal against the file-group count the index actually has. It also looks at the + * largest file group directly, since a group past the configured maximum size is the symptom the + * operator will feel regardless of how the count came out. Both signals share one remedy: the index + * has to be re-bootstrapped with corrected sizing, which is disruptive and is spelled out in the + * findings. + * + * <p>A partitioned record index is sized per data partition, not as a whole, so for that layout + * the count comparison is made per data partition and the worst one is reported. Which layout the + * writer sizes for is read from its configuration and which layout is on storage is read from the + * index's file-group names; if the two disagree, the count comparison is skipped and the report says + * so, while the file-group size rule still runs. + * + * <p>The record count is estimated from the index's own footprint on storage divided by the same + * average record size the writer sizes with, so that no data file is scanned. The estimate is + * conservative -- index files are compressed, so the true count is higher -- and it is labelled as + * an estimate in the report. + */ +public class RecordIndexSizingCheck implements HealthCheck { + + static final String NAME = "record-index"; + + /** + * Actual-to-ideal file-group ratio below which the index is called undersized. The ideal count + * already carries the writer's growth factor, so an index initialized on a smaller table sits + * somewhat below 1.0 during ordinary growth. The slack keeps that from alerting while a table + * that has grown past what its index was sized for still does. + */ + static final double MIN_ACCEPTABLE_FILE_GROUP_RATIO = 0.5; + + /** + * How far past the configured maximum a single file group may grow before the index is called + * unhealthy. File groups are bounded only at initialization; between metadata compactions, log + * files push a group past the configured size transiently, so alerting at exactly the maximum + * would fire on healthy tables. + */ + static final double MAX_FILE_GROUP_SIZE_SLACK_FACTOR = 1.5; + + @Override + public String getName() { + return NAME; + } + + @Override + public String getDescription() { + return "Whether the record-level index has enough file groups for the table it serves"; + } + + @Override + public boolean appliesTo(HealthCheckContext context) { + return context.getMetaClient().getTableConfig().isMetadataPartitionAvailable(MetadataPartitionType.RECORD_INDEX); + } + + @Override + public HealthCheckResult check(HealthCheckContext context) throws Exception { + if (!appliesTo(context)) { + return HealthCheckResult.skipped(NAME, String.format( + "Table has no record-level index: '%s' in the table configuration does not list '%s'", + HoodieTableConfig.TABLE_METADATA_PARTITIONS.key(), MetadataPartitionType.RECORD_INDEX.getPartitionPath())); + } + + TypedProperties props = context.getProps(); + boolean partitionedIndex = ConfigUtils.getBooleanWithAltKeys(props, HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP); Review Comment: 🤖 `RECORD_LEVEL_INDEX_ENABLE_PROP` picks which keys are governing, but when it isn't supplied it silently falls back to `false`. On a partitioned-index table, `--apply-all-defaults` then never runs the count comparison, and without it the skip message asks for the global keys. Since the on-storage layout is already detected, could we use it when the flag isn't supplied? <sub><i>⚠️ AI-generated; verify before applying. React 👍/👎 to flag quality.</i></sub> -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
