codope commented on code in PR #20154: URL: https://github.com/apache/hudi/pull/20154#discussion_r4172236307
########## hudi-utilities/src/main/java/org/apache/hudi/utilities/health/MetadataTableSyncCheck.java: ########## @@ -0,0 +1,365 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.utilities.health; + +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.engine.HoodieLocalEngineContext; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieBaseFile; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.table.view.HoodieTableFileSystemView; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.metadata.FileSystemBackedTableMetadata; +import org.apache.hudi.metadata.HoodieBackedTableMetadata; +import org.apache.hudi.metadata.HoodieTableMetadata; +import org.apache.hudi.storage.StoragePath; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; + +/** + * Checks whether the metadata table is caught up with the data table, and whether the file listing + * it serves agrees with what is actually on storage. + * + * <p>Two signals. The first is sync lag: every completed data-table write is mirrored by a delta + * commit on the metadata table with the same instant time, so the latest completed metadata-table + * delta commit says how far the metadata table has been brought up to date. A data-table write + * newer than that is one the metadata table does not know about, and readers relying on the + * metadata table for listings will not see its files. + * + * <p>The second is listing consistency: for a bounded sample of partitions, the latest base files + * and log files are listed twice -- once through the metadata table and once straight from the + * file system -- and compared. A disagreement means the metadata table has drifted from storage, + * which is the one condition that justifies rebuilding it. The sample is bounded because listing a + * partition from the file system is precisely the expensive operation the metadata table exists to + * avoid; a daily health check must not turn into a full table scan. It is weighted towards the + * most recently written partitions, where drift actually originates, with a couple of the oldest + * added for coverage. For a complete comparison the finding points at + * {@code HoodieMetadataTableValidator}. + * + * <p>Engine-agnostic: both listing backends and the file-system views are built on + * {@link HoodieLocalEngineContext}, so no Spark session is needed. + */ +public class MetadataTableSyncCheck implements HealthCheck { + + static final String NAME = "mdt-sync"; + + /** + * Upper bound on partitions whose listings are compared. Each sampled partition costs one + * file-system listing, which is the operation the metadata table exists to avoid, so the sample + * is kept small enough to be cheap on a schedule while still catching drift that affects the + * table broadly rather than a single partition. + * + * <p>The sample is taken from the <em>end</em> of the sorted partition list. Drift comes from + * recent writes, cleans and rollbacks, so the partitions most likely to disagree are the ones + * most recently written -- and on the common date-partitioned table, lexical order is time + * order, so the last partitions are the newest. Sampling the first ones would compare the + * partitions least likely to have changed. + */ + static final int MAX_PARTITIONS_TO_COMPARE = 20; + + /** + * How many of the oldest partitions are added to the sample when the table has more partitions + * than {@link #MAX_PARTITIONS_TO_COMPARE}, so that drift confined to old data (a bad clean, a + * partition dropped by hand) is not invisible to a check that otherwise looks only at new data. + */ + static final int OLDEST_PARTITIONS_TO_INCLUDE = 2; + + /** + * How many differing file names to spell out per partition before truncating. Enough to identify + * the pattern (one file, one file group, everything) without flooding the report. + */ + static final int MAX_DIFFERING_FILES_TO_NAME = 5; + + @Override + public String getName() { + return NAME; + } + + @Override + public String getDescription() { + return "Whether the metadata table is caught up with the data table and its file listing matches storage"; + } + + @Override + public HealthCheckResult check(HealthCheckContext context) throws Exception { + Option<String> missing = context.firstMissingConfig(HoodieMetadataConfig.ENABLE.key()); + if (missing.isPresent()) { + return context.skipForMissingConfig(NAME, missing.get()); + } + + boolean metadataEnabled = Boolean.parseBoolean(context.getProps().getString( + HoodieMetadataConfig.ENABLE.key(), String.valueOf(HoodieMetadataConfig.ENABLE.defaultValue()))); + if (!metadataEnabled) { + return HealthCheckResult.skipped(NAME, "Metadata table is disabled for this writer (" + + HoodieMetadataConfig.ENABLE.key() + "=false), so there is nothing to compare against"); + } + + HoodieTableMetaClient metaClient = context.getMetaClient(); + Option<String> noMetadataTable = MetadataTablePresence.reasonAbsent(metaClient); + if (noMetadataTable.isPresent()) { + return HealthCheckResult.skipped(NAME, noMetadataTable.get()); + } + + HealthCheckResult.Builder result = HealthCheckResult.builder(NAME) + .config(HoodieMetadataConfig.ENABLE.key(), true) + .config("effective.partitions.sampled.max", MAX_PARTITIONS_TO_COMPARE); + + HoodieEngineContext engineContext = new HoodieLocalEngineContext(metaClient.getStorageConf()); + String basePath = metaClient.getBasePath().toString(); + HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().enable(true).build(); + + try (HoodieTableMetadata metadataBacked = + new HoodieBackedTableMetadata(engineContext, metaClient.getStorage(), metadataConfig, basePath); + HoodieTableMetadata fileSystemBacked = + new FileSystemBackedTableMetadata(engineContext, metaClient.getTableConfig(), metaClient.getStorage(), basePath)) { + + SyncLag lag = measureSyncLag(metaClient, metadataBacked); + lag.report(result); + + ListingComparison listing = compareSampledPartitions(metaClient, metadataBacked, fileSystemBacked); + listing.report(result); + + return verdict(result, lag, listing); + } + } + + private HealthCheckResult verdict(HealthCheckResult.Builder result, SyncLag lag, ListingComparison listing) { + boolean behind = lag.commitsBehind > 0; + boolean inconsistent = !listing.differences.isEmpty(); + + if (!behind && !inconsistent) { + return result.status(HealthStatus.HEALTHY) + .summary(String.format("Metadata table is synced to %s and its listing matches storage in all %d sampled partition(s)", + lag.syncedInstant.orElse("(no data-table writes yet)"), listing.partitionsSampled)) + .build(); + } + + if (behind) { + result.finding(String.format( + "The metadata table is synced to %s but the data table's latest completed write is %s; %d completed " + + "data-table write(s) are missing from the metadata table. Readers that list through the " + + "metadata table will not see files from those writes.", + lag.syncedInstant.orElse("nothing"), lag.latestDataInstant.orElse("nothing"), lag.commitsBehind)); + result.finding("If a writer is running right now, this can be a write that is still being finalized: " + + "rerun once it completes. If the metadata table stays behind, look for failed or incomplete " + + "metadata-table delta commits ('metadata timeline show incomplete' in hudi-cli) and for " + + "metadata-table errors in the writer logs. Do not delete or rebuild the metadata table for " + + "lag alone; the next successful write brings it up to date."); Review Comment: Reproduced, and readers lose data. I made one write with the metadata table disabled (into a new partition), then one normal write. This check then says HEALTHY, but reading through the metadata table returns 20 rows against 41 on storage. The new partition is never sampled, because the sample comes from the metadata table's own partition list. Could lag be treated as drift, and could the sample include partitions found on storage? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
