nsivabalan commented on code in PR #20153: URL: https://github.com/apache/hudi/pull/20153#discussion_r4279973158
########## hudi-utilities/src/main/java/org/apache/hudi/utilities/health/RecordIndexSizingCheck.java: ########## @@ -0,0 +1,420 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.utilities.health; + +import org.apache.hudi.common.config.ConfigProperty; +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.engine.HoodieLocalEngineContext; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.ConfigUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.metadata.HoodieBackedTableMetadata; +import org.apache.hudi.metadata.HoodieTableMetadataUtil; +import org.apache.hudi.metadata.MetadataPartitionType; + +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static org.apache.hudi.metadata.HoodieTableMetadataUtil.RECORD_INDEX_AVERAGE_RECORD_SIZE; + +/** + * Checks whether the record-level index is sized appropriately for the table it serves. + * + * <p>The number of file groups in the record index is decided once, when the index is initialized, + * from the record count at that moment and the writer's sizing configuration. It never changes + * afterwards: every subsequent index write lands in one of those same file groups. A table that has + * grown well past what its index was sized for therefore has file groups that are each far larger + * than the writer's configuration intended, which slows every index lookup and concentrates every + * index write on the same few file groups. + * + * <p>So the check re-runs the writer's own sizing estimate against the table as it is now and + * compares that ideal against the file-group count the index actually has. It also looks at the + * largest file group directly, since a group past the configured maximum size is the symptom the + * operator will feel regardless of how the count came out. Both signals share one remedy: the index + * has to be re-bootstrapped with corrected sizing, which is disruptive and is spelled out in the + * findings. + * + * <p>A partitioned record index is sized per data partition, not as a whole, so for that layout + * the count comparison is made per data partition and the worst one is reported. Which layout the + * writer sizes for is read from its configuration and which layout is on storage is read from the + * index's file-group names; if the two disagree, the count comparison is skipped and the report says + * so, while the file-group size rule still runs. + * + * <p>The record count is estimated from the index's own footprint on storage divided by the same + * average record size the writer sizes with, so that no data file is scanned. The estimate is + * conservative -- index files are compressed, so the true count is higher -- and it is labelled as + * an estimate in the report. + */ +public class RecordIndexSizingCheck implements HealthCheck { + + static final String NAME = "record-index"; + + /** + * Actual-to-ideal file-group ratio below which the index is called undersized. The ideal count + * already carries the writer's growth factor, so an index initialized on a smaller table sits + * somewhat below 1.0 during ordinary growth. The slack keeps that from alerting while a table + * that has grown past what its index was sized for still does. + */ + static final double MIN_ACCEPTABLE_FILE_GROUP_RATIO = 0.5; + + /** + * How far past the configured maximum a single file group may grow before the index is called + * unhealthy. File groups are bounded only at initialization; between metadata compactions, log + * files push a group past the configured size transiently, so alerting at exactly the maximum + * would fire on healthy tables. + */ + static final double MAX_FILE_GROUP_SIZE_SLACK_FACTOR = 1.5; + + @Override + public String getName() { + return NAME; + } + + @Override + public String getDescription() { + return "Whether the record-level index has enough file groups for the table it serves"; + } + + @Override + public boolean appliesTo(HealthCheckContext context) { + return context.getMetaClient().getTableConfig().isMetadataPartitionAvailable(MetadataPartitionType.RECORD_INDEX); + } + + @Override + public HealthCheckResult check(HealthCheckContext context) throws Exception { + if (!appliesTo(context)) { + return HealthCheckResult.skipped(NAME, String.format( + "Table has no record-level index: '%s' in the table configuration does not list '%s'", + HoodieTableConfig.TABLE_METADATA_PARTITIONS.key(), MetadataPartitionType.RECORD_INDEX.getPartitionPath())); + } + + TypedProperties props = context.getProps(); + boolean partitionedIndex = ConfigUtils.getBooleanWithAltKeys(props, HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP); + ConfigProperty<Integer> minCountProp = partitionedIndex + ? HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP + : HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP; + ConfigProperty<Integer> maxCountProp = partitionedIndex + ? HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP + : HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP; + + Option<String> missing = firstMissingConfig(context, minCountProp, maxCountProp, + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP, + HoodieMetadataConfig.RECORD_INDEX_GROWTH_FACTOR_PROP); + if (missing.isPresent()) { + return context.skipForMissingConfig(NAME, missing.get()); + } + + SizingConfig sizing = new SizingConfig( + Integer.parseInt(configValue(props, minCountProp)), + Integer.parseInt(configValue(props, maxCountProp)), + Float.parseFloat(configValue(props, HoodieMetadataConfig.RECORD_INDEX_GROWTH_FACTOR_PROP)), + Long.parseLong(configValue(props, HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP))); + + HealthCheckResult.Builder result = HealthCheckResult.builder(NAME) + .config(HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key(), partitionedIndex) + .config(minCountProp.key(), sizing.minFileGroupCount) + .config(maxCountProp.key(), sizing.maxFileGroupCount) + .config(HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key(), sizing.maxFileGroupSizeBytes) + .config(HoodieMetadataConfig.RECORD_INDEX_GROWTH_FACTOR_PROP.key(), sizing.growthFactor) + .config("effective.average.record.size.bytes", RECORD_INDEX_AVERAGE_RECORD_SIZE); + + if (sizing.maxFileGroupSizeBytes < RECORD_INDEX_AVERAGE_RECORD_SIZE) { + return result.status(HealthStatus.SKIPPED) + .summary(String.format( + "'%s' is %d bytes, smaller than the %d-byte average index record the writer sizes with, so no " + + "file-group count can be estimated. The writer cannot size an index with this value either.", + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key(), sizing.maxFileGroupSizeBytes, + RECORD_INDEX_AVERAGE_RECORD_SIZE)) + .build(); + } + + RecordIndexFootprint footprint = measureRecordIndex(context.getMetaClient()); + long maxHealthyFileGroupBytes = (long) (MAX_FILE_GROUP_SIZE_SLACK_FACTOR * sizing.maxFileGroupSizeBytes); + result.config("effective.max.healthy.file.group.bytes", maxHealthyFileGroupBytes) + .config("observed.file.group.count", footprint.fileGroupCount) + .config("observed.base.file.count", footprint.baseFileCount) + .config("observed.log.file.count", footprint.logFileCount) + .config("observed.total.size.bytes", footprint.totalSizeBytes) + .config("observed.largest.file.group.bytes", footprint.largestFileGroupBytes) + .config("observed.largest.file.group.id", footprint.largestFileGroupId); + + Option<CountComparison> comparison = compareFileGroupCount(footprint, partitionedIndex, sizing, result); + boolean undersized = comparison.isPresent() && comparison.get().isUndersized(); + boolean oversizedFileGroup = footprint.largestFileGroupBytes > maxHealthyFileGroupBytes; + + if (!undersized && !oversizedFileGroup) { + return result.status(HealthStatus.HEALTHY) + .summary(comparison.isPresent() + ? String.format("%s has %d file group(s) against an ideal of %d for an estimated %d records; largest " + + "file group is %d bytes", comparison.get().subject(), comparison.get().observedFileGroupCount, + comparison.get().idealFileGroupCount, comparison.get().estimatedRecordCount, footprint.largestFileGroupBytes) + : String.format("Record index has %d file group(s); largest file group is %d bytes, within the " + + "configured maximum. File-group count was not compared against the writer's sizing.", + footprint.fileGroupCount, footprint.largestFileGroupBytes)) + .build(); + } + + result.status(HealthStatus.UNHEALTHY); + if (undersized) { + CountComparison shortfall = comparison.get(); + result.summary(String.format( + "%s has %d file group(s) but the writer's sizing calls for %d (at least %d to be healthy)", + shortfall.subject(), shortfall.observedFileGroupCount, shortfall.idealFileGroupCount, + shortfall.minHealthyFileGroupCount())) + .finding(String.format( + "%s was initialized with %d file group(s). Its %d bytes on storage put it at an estimated %d records " + + "(an estimate: index bytes divided by the %d-byte average record the writer sizes with), which " + + "with the configured growth factor and maximum file-group size calls for %d. Each file group " + + "holds roughly %.1fx more keys than intended, so index lookups read larger files and every " + + "index write concentrates on the same few file groups.", + shortfall.subject(), shortfall.observedFileGroupCount, shortfall.observedSizeBytes, + shortfall.estimatedRecordCount, RECORD_INDEX_AVERAGE_RECORD_SIZE, shortfall.idealFileGroupCount, + (double) shortfall.idealFileGroupCount / Math.max(shortfall.observedFileGroupCount, 1))); + } + if (oversizedFileGroup) { + if (!undersized) { + result.summary(String.format( + "Record index file group %s is %d bytes, past the configured maximum of %d bytes", + footprint.largestFileGroupId, footprint.largestFileGroupBytes, sizing.maxFileGroupSizeBytes)); + } + result.finding(String.format( + "File group %s is %d bytes, past the configured maximum of %d bytes even with %.1fx slack. Compacting " + + "it takes proportionally longer and every lookup routed to it reads that much more. A file group " + + "that keeps growing between metadata compactions means the index has too few file groups for " + + "the table.", + footprint.largestFileGroupId, footprint.largestFileGroupBytes, sizing.maxFileGroupSizeBytes, + MAX_FILE_GROUP_SIZE_SLACK_FACTOR)); + } + result.finding(rebootstrapRemedy()); + return result.build(); + } + + /** + * Compares the file-group count against the writer's sizing estimate, at the granularity the + * writer sizes at: the whole index for a global record index, each data partition for a + * partitioned one (the worst data partition is returned). Echoes what it compared, or why it + * did not, into the result. Empty when the writer's configured layout disagrees with the layout + * on storage, since neither estimate would be comparable to the observed count then. + */ + private static Option<CountComparison> compareFileGroupCount(RecordIndexFootprint footprint, boolean partitionedIndex, + SizingConfig sizing, HealthCheckResult.Builder result) { + boolean layoutKnown = footprint.fileGroupCount > 0; + if (layoutKnown && footprint.partitionedLayout != partitionedIndex) { + result.config("effective.count.comparison", String.format( + "skipped: writer configuration says a %s record index but the index on storage is %s", + partitionedIndex ? "partitioned" : "global", footprint.partitionedLayout ? "partitioned" : "global")); + return Option.empty(); + } + + CountComparison comparison; + if (!partitionedIndex) { + result.config("effective.count.comparison", "whole index (global record index)"); + comparison = CountComparison.of(Option.empty(), footprint, sizing); + } else { + result.config("effective.count.comparison", "per data partition (partitioned record index); worst data partition reported") + .config("observed.data.partition.count", footprint.byDataPartition.size()); + comparison = CountComparison.of(Option.empty(), footprint, sizing); + for (Map.Entry<String, RecordIndexFootprint> entry : footprint.byDataPartition.entrySet()) { + CountComparison candidate = CountComparison.of(Option.of(entry.getKey()), entry.getValue(), sizing); + if (candidate.ratio() < comparison.ratio() || comparison.dataPartition.isEmpty()) { + comparison = candidate; + } + } + if (comparison.dataPartition.isPresent()) { + result.config("observed.worst.data.partition", comparison.dataPartition.get()) + .config("observed.worst.data.partition.file.group.count", comparison.observedFileGroupCount); + } + } + + result.config("effective.ideal.file.group.count", comparison.idealFileGroupCount) + .config("effective.min.healthy.file.group.count", comparison.minHealthyFileGroupCount()) + .config("observed.estimated.record.count", comparison.estimatedRecordCount); + return Option.of(comparison); + } + + /** + * The one remedy for a mis-sized record index. The file-group count is fixed at initialization, + * so there is no in-place resize; the operator has to drop the index and build it again. + */ + private static String rebootstrapRemedy() { + return "The file-group count of the record index is fixed when the index is initialized and cannot be " + + "changed in place. To resize it, re-bootstrap the index: drop it with 'metadata delete-record-index' in " + + "hudi-cli (or HoodieIndexer --mode dropindex --index-types RECORD_INDEX), correct the sizing " + + "configuration, then rebuild it with HoodieIndexer --mode scheduleAndExecute --index-types RECORD_INDEX. " Review Comment: Good catch, and it turns out to be worse than the remedy text — I filed apache/hudi#20343. `HoodieIndexer` cannot rebuild a partitioned record-level index at all. At `HoodieIndexer.java:188-189` it maps the on-storage partition name `record_index` to `hoodie.metadata.global.record.level.index.enable` unconditionally, and `RECORD_LEVEL_INDEX_ENABLE_PROP` is never imported or referenced in the class. Both variants share the same `record_index` partition name, so the partition name alone cannot tell them apart. Passing the key in the indexer's own `--props` does not help either: that `props.setProperty` runs *after* the user's props are loaded, so it overwrites them. There is no configuration workaround. This predates this PR — introduced in 80381808a6c3 (#14180, Nov 2025), the commit that split the enable flag into global and partitioned variants and updated the indexer to the global constant only. It shipped in 1.2.0. So rather than suggest a command that silently produces the wrong index shape, the remedy will say the partitioned case cannot currently be rebuilt this way and link #20343. Also taking your note that `delete-record-index` backs up by default — the wording implied otherwise. ########## hudi-utilities/src/main/java/org/apache/hudi/utilities/health/RecordIndexSizingCheck.java: ########## @@ -0,0 +1,420 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.utilities.health; + +import org.apache.hudi.common.config.ConfigProperty; +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.config.TypedProperties; +import org.apache.hudi.common.engine.HoodieLocalEngineContext; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.ConfigUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.metadata.HoodieBackedTableMetadata; +import org.apache.hudi.metadata.HoodieTableMetadataUtil; +import org.apache.hudi.metadata.MetadataPartitionType; + +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static org.apache.hudi.metadata.HoodieTableMetadataUtil.RECORD_INDEX_AVERAGE_RECORD_SIZE; + +/** + * Checks whether the record-level index is sized appropriately for the table it serves. + * + * <p>The number of file groups in the record index is decided once, when the index is initialized, + * from the record count at that moment and the writer's sizing configuration. It never changes + * afterwards: every subsequent index write lands in one of those same file groups. A table that has + * grown well past what its index was sized for therefore has file groups that are each far larger + * than the writer's configuration intended, which slows every index lookup and concentrates every + * index write on the same few file groups. + * + * <p>So the check re-runs the writer's own sizing estimate against the table as it is now and + * compares that ideal against the file-group count the index actually has. It also looks at the + * largest file group directly, since a group past the configured maximum size is the symptom the + * operator will feel regardless of how the count came out. Both signals share one remedy: the index + * has to be re-bootstrapped with corrected sizing, which is disruptive and is spelled out in the + * findings. + * + * <p>A partitioned record index is sized per data partition, not as a whole, so for that layout + * the count comparison is made per data partition and the worst one is reported. Which layout the + * writer sizes for is read from its configuration and which layout is on storage is read from the + * index's file-group names; if the two disagree, the count comparison is skipped and the report says + * so, while the file-group size rule still runs. + * + * <p>The record count is estimated from the index's own footprint on storage divided by the same + * average record size the writer sizes with, so that no data file is scanned. The estimate is + * conservative -- index files are compressed, so the true count is higher -- and it is labelled as + * an estimate in the report. + */ +public class RecordIndexSizingCheck implements HealthCheck { + + static final String NAME = "record-index"; + + /** + * Actual-to-ideal file-group ratio below which the index is called undersized. The ideal count + * already carries the writer's growth factor, so an index initialized on a smaller table sits + * somewhat below 1.0 during ordinary growth. The slack keeps that from alerting while a table + * that has grown past what its index was sized for still does. + */ + static final double MIN_ACCEPTABLE_FILE_GROUP_RATIO = 0.5; + + /** + * How far past the configured maximum a single file group may grow before the index is called + * unhealthy. File groups are bounded only at initialization; between metadata compactions, log + * files push a group past the configured size transiently, so alerting at exactly the maximum + * would fire on healthy tables. + */ + static final double MAX_FILE_GROUP_SIZE_SLACK_FACTOR = 1.5; + + @Override + public String getName() { + return NAME; + } + + @Override + public String getDescription() { + return "Whether the record-level index has enough file groups for the table it serves"; + } + + @Override + public boolean appliesTo(HealthCheckContext context) { + return context.getMetaClient().getTableConfig().isMetadataPartitionAvailable(MetadataPartitionType.RECORD_INDEX); + } + + @Override + public HealthCheckResult check(HealthCheckContext context) throws Exception { + if (!appliesTo(context)) { + return HealthCheckResult.skipped(NAME, String.format( + "Table has no record-level index: '%s' in the table configuration does not list '%s'", + HoodieTableConfig.TABLE_METADATA_PARTITIONS.key(), MetadataPartitionType.RECORD_INDEX.getPartitionPath())); + } + + TypedProperties props = context.getProps(); + boolean partitionedIndex = ConfigUtils.getBooleanWithAltKeys(props, HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP); + ConfigProperty<Integer> minCountProp = partitionedIndex + ? HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP + : HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP; + ConfigProperty<Integer> maxCountProp = partitionedIndex + ? HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP + : HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP; + + Option<String> missing = firstMissingConfig(context, minCountProp, maxCountProp, + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP, + HoodieMetadataConfig.RECORD_INDEX_GROWTH_FACTOR_PROP); + if (missing.isPresent()) { + return context.skipForMissingConfig(NAME, missing.get()); + } + + SizingConfig sizing = new SizingConfig( + Integer.parseInt(configValue(props, minCountProp)), + Integer.parseInt(configValue(props, maxCountProp)), + Float.parseFloat(configValue(props, HoodieMetadataConfig.RECORD_INDEX_GROWTH_FACTOR_PROP)), + Long.parseLong(configValue(props, HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP))); + + HealthCheckResult.Builder result = HealthCheckResult.builder(NAME) + .config(HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key(), partitionedIndex) + .config(minCountProp.key(), sizing.minFileGroupCount) + .config(maxCountProp.key(), sizing.maxFileGroupCount) + .config(HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key(), sizing.maxFileGroupSizeBytes) + .config(HoodieMetadataConfig.RECORD_INDEX_GROWTH_FACTOR_PROP.key(), sizing.growthFactor) + .config("effective.average.record.size.bytes", RECORD_INDEX_AVERAGE_RECORD_SIZE); + + if (sizing.maxFileGroupSizeBytes < RECORD_INDEX_AVERAGE_RECORD_SIZE) { + return result.status(HealthStatus.SKIPPED) + .summary(String.format( + "'%s' is %d bytes, smaller than the %d-byte average index record the writer sizes with, so no " + + "file-group count can be estimated. The writer cannot size an index with this value either.", + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key(), sizing.maxFileGroupSizeBytes, + RECORD_INDEX_AVERAGE_RECORD_SIZE)) + .build(); + } + + RecordIndexFootprint footprint = measureRecordIndex(context.getMetaClient()); + long maxHealthyFileGroupBytes = (long) (MAX_FILE_GROUP_SIZE_SLACK_FACTOR * sizing.maxFileGroupSizeBytes); + result.config("effective.max.healthy.file.group.bytes", maxHealthyFileGroupBytes) + .config("observed.file.group.count", footprint.fileGroupCount) + .config("observed.base.file.count", footprint.baseFileCount) + .config("observed.log.file.count", footprint.logFileCount) + .config("observed.total.size.bytes", footprint.totalSizeBytes) + .config("observed.largest.file.group.bytes", footprint.largestFileGroupBytes) + .config("observed.largest.file.group.id", footprint.largestFileGroupId); + + Option<CountComparison> comparison = compareFileGroupCount(footprint, partitionedIndex, sizing, result); + boolean undersized = comparison.isPresent() && comparison.get().isUndersized(); + boolean oversizedFileGroup = footprint.largestFileGroupBytes > maxHealthyFileGroupBytes; + + if (!undersized && !oversizedFileGroup) { + return result.status(HealthStatus.HEALTHY) + .summary(comparison.isPresent() + ? String.format("%s has %d file group(s) against an ideal of %d for an estimated %d records; largest " + + "file group is %d bytes", comparison.get().subject(), comparison.get().observedFileGroupCount, + comparison.get().idealFileGroupCount, comparison.get().estimatedRecordCount, footprint.largestFileGroupBytes) + : String.format("Record index has %d file group(s); largest file group is %d bytes, within the " + + "configured maximum. File-group count was not compared against the writer's sizing.", + footprint.fileGroupCount, footprint.largestFileGroupBytes)) + .build(); + } + + result.status(HealthStatus.UNHEALTHY); + if (undersized) { + CountComparison shortfall = comparison.get(); + result.summary(String.format( + "%s has %d file group(s) but the writer's sizing calls for %d (at least %d to be healthy)", + shortfall.subject(), shortfall.observedFileGroupCount, shortfall.idealFileGroupCount, + shortfall.minHealthyFileGroupCount())) + .finding(String.format( + "%s was initialized with %d file group(s). Its %d bytes on storage put it at an estimated %d records " + + "(an estimate: index bytes divided by the %d-byte average record the writer sizes with), which " + + "with the configured growth factor and maximum file-group size calls for %d. Each file group " + + "holds roughly %.1fx more keys than intended, so index lookups read larger files and every " Review Comment: Leaving this one as is, and I want to be straight that it is a judgement call rather than a disagreement with the arithmetic. You are right that `ideal / observed` carries the 2.0 growth factor, so the multiplier reads higher than the present overshoot: a 1.26× overshoot surfacing as "2.5x more keys" is exactly that factor showing through. `averageGroupBytes / maxFileGroupSize` would describe today's state more precisely. The reason I am not changing it: the check exists to decide whether an index needs re-bootstrapping, and that decision is about where the index is heading, not where it is now — a file group at 1.26× today with the growth factor applied is the case the check is meant to catch early. Switching the ratio to present-state bytes would make the number more accurate and the verdict less useful, and I would rather not split them across two patches. What is fair is that the wording does not say which it is. If the phrasing made clear it is a projected rather than current multiple, would that address it? Happy to reword rather than leave it ambiguous. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
