amogh-jahagirdar commented on code in PR #17622: URL: https://github.com/apache/iceberg/pull/17622#discussion_r4568708448
########## spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RepairTableSparkAction.java: ########## @@ -0,0 +1,905 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.actions; + +import static org.apache.iceberg.MetadataTableType.ENTRIES; + +import java.io.Serializable; +import java.util.EnumMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.hadoop.fs.Path; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.GenericManifestFile; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManifestContent; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.ManifestWriter; +import org.apache.iceberg.Metrics; +import org.apache.iceberg.MetricsConfig; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Partitioning; +import org.apache.iceberg.RewriteManifests; +import org.apache.iceberg.RollingManifestWriter; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotSummary; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.actions.ImmutableRepairTable; +import org.apache.iceberg.actions.RepairTable; +import org.apache.iceberg.encryption.EncryptedOutputFile; +import org.apache.iceberg.encryption.EncryptingFileIO; +import org.apache.iceberg.exceptions.CleanableFailure; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.io.SupportsBulkOperations; +import org.apache.iceberg.mapping.NameMapping; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.spark.JobGroupInfo; +import org.apache.iceberg.spark.SparkContentFile; +import org.apache.iceberg.spark.SparkDataFile; +import org.apache.iceberg.spark.SparkDeleteFile; +import org.apache.iceberg.spark.source.SerializableTableWithSize; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.PropertyUtil; +import org.apache.iceberg.util.ScanTaskUtil; +import org.apache.iceberg.util.Tasks; +import org.apache.iceberg.util.ThreadPools; +import org.apache.spark.api.java.function.MapPartitionsFunction; +import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoder; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.functions; +import org.apache.spark.sql.types.StructType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; + +/** + * An action that repairs incorrect statistics in the manifests of a table, rewriting only the + * manifests that contain an incorrect entry. + * + * <p>Deletion vectors are not repaired. NaN counts and floating-point and geometry bounds are kept + * as stored, because file footers cannot reconstruct them reliably. + */ +public class RepairTableSparkAction extends BaseSnapshotUpdateSparkAction<RepairTableSparkAction> + implements RepairTable { + + public static final String USE_CACHING = "use-caching"; + public static final boolean USE_CACHING_DEFAULT = false; + + /** + * Whether to also repair column statistics, not only record counts and file sizes. Disabled by + * default: the metrics config a file was written under is not recorded, so correct column + * statistics can differ from the recomputed ones after the table's config changes. + */ + public static final String REPAIR_COLUMN_METRICS = "repair-column-metrics"; + + public static final boolean REPAIR_COLUMN_METRICS_DEFAULT = false; + + private static final Logger LOG = LoggerFactory.getLogger(RepairTableSparkAction.class); + + private static final RepairTable.Result EMPTY_RESULT = + ImmutableRepairTable.Result.builder() + .repairedManifests(ImmutableList.of()) + .repairedEntryCount(0L) + .build(); + + private static final String NEW_MANIFEST_PREFIX = "repaired-m-"; + + private final Table table; + private final int formatVersion; + private final long targetManifestSizeBytes; + private final boolean shouldStageManifests; + private final String outputLocation; + + private boolean repairFileMetrics = false; + private boolean dryRun = false; + + RepairTableSparkAction(SparkSession spark, Table table) { + super(spark); + this.table = table; + this.targetManifestSizeBytes = + PropertyUtil.propertyAsLong( + table.properties(), + TableProperties.MANIFEST_TARGET_SIZE_BYTES, + TableProperties.MANIFEST_TARGET_SIZE_BYTES_DEFAULT); + + TableOperations ops = ((HasTableOperations) table).operations(); + Path metadataFilePath = new Path(ops.metadataFileLocation("file")); + this.outputLocation = metadataFilePath.getParent().toString(); + this.formatVersion = ops.current().formatVersion(); + + boolean snapshotIdInheritanceEnabled = + PropertyUtil.propertyAsBoolean( + table.properties(), + TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED, + TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED_DEFAULT); + this.shouldStageManifests = formatVersion == 1 && !snapshotIdInheritanceEnabled; + } + + @Override + protected RepairTableSparkAction self() { + return this; + } + + @Override + public RepairTableSparkAction repairFileMetrics() { + this.repairFileMetrics = true; + return this; + } + + @Override + public RepairTableSparkAction dryRun() { + this.dryRun = true; + return this; + } + + @Override + public RepairTable.Result execute() { + if (!repairFileMetrics) { + return EMPTY_RESULT; + } + + String desc = String.format("Repairing manifests in %s (dryRun=%s)", table.name(), dryRun); + JobGroupInfo info = newJobGroupInfo("REPAIR-TABLE", desc); + return withJobGroupInfo(info, this::doExecute); + } + + private RepairTable.Result doExecute() { + Snapshot currentSnapshot = table.currentSnapshot(); + if (currentSnapshot == null) { + return EMPTY_RESULT; + } + + List<ManifestFile> repairedManifests = Lists.newArrayList(); + List<ManifestFile> newManifests = Lists.newArrayList(); + long repairedCount = 0L; + + try { + for (ManifestContent content : ManifestContent.values()) { + RepairedManifests repaired = repairTable(content, currentSnapshot); + repairedManifests.addAll(repaired.repairedManifests()); + newManifests.addAll(repaired.newManifests()); + repairedCount += repaired.repairedCount(); + } + } catch (Exception e) { + // the manifests of earlier groups are not committed, so delete them rather than orphan them + deleteFiles(Iterables.transform(newManifests, ManifestFile::path)); + throw e; + } + + if (repairedManifests.isEmpty()) { + return EMPTY_RESULT; + } + + if (!dryRun) { + replaceManifests(repairedManifests, newManifests); + } + + LOG.info( + "Repaired the stats of {} manifest entries, rewriting {} manifests as {} (dryRun={})", + repairedCount, + repairedManifests.size(), + newManifests.size(), + dryRun); + + return ImmutableRepairTable.Result.builder() + .repairedManifests(repairedManifests) + .repairedEntryCount(repairedCount) + .build(); + } + + private RepairedManifests repairTable(ManifestContent content, Snapshot snapshot) { + List<ManifestFile> manifests = loadManifests(content, snapshot); + if (manifests.isEmpty()) { + return RepairedManifests.empty(); + } + + // rewriting a manifest with a spec other than its own would change its partition data + Map<Integer, List<ManifestFile>> manifestsBySpecId = + manifests.stream().collect(Collectors.groupingBy(ManifestFile::partitionSpecId)); + + List<ManifestFile> repairedManifests = Lists.newArrayList(); + List<ManifestFile> newManifests = Lists.newArrayList(); + long repairedCount = 0L; + + try { + for (Map.Entry<Integer, List<ManifestFile>> group : manifestsBySpecId.entrySet()) { + RepairedManifests repaired = repairManifests(content, group.getKey(), group.getValue()); + repairedManifests.addAll(repaired.repairedManifests()); + newManifests.addAll(repaired.newManifests()); + repairedCount += repaired.repairedCount(); + } + } catch (Exception e) { + // the manifests of earlier groups are not committed, so delete them rather than orphan them + deleteFiles(Iterables.transform(newManifests, ManifestFile::path)); + throw e; + } + + return RepairedManifests.of(repairedManifests, newManifests, repairedCount); + } + + private RepairedManifests repairManifests( + ManifestContent content, int specId, List<ManifestFile> manifests) { + Dataset<Row> entryDF = buildManifestEntryDF(manifests); + + return withReusableDS( + entryDF, + df -> { + // cached because it is read several times and recomputing it would read every file again + Dataset<Row> verdicts = + df.mapPartitions( + new CheckStats(newRepairContext(content)), + Encoders.tuple(Encoders.STRING(), Encoders.STRING())) + .toDF("manifest", "path") + .cache(); + + try { + Set<String> manifestsToRewrite = + Sets.newHashSet( + verdicts.select("manifest").distinct().as(Encoders.STRING()).collectAsList()); + + if (manifestsToRewrite.isEmpty()) { + return RepairedManifests.empty(); + } + + long repairedCount = verdicts.count(); + List<ManifestFile> rewritten = + manifests.stream() + .filter(manifest -> manifestsToRewrite.contains(manifest.path())) + .collect(Collectors.toList()); + + if (dryRun) { + return RepairedManifests.of(rewritten, ImmutableList.of(), repairedCount); + } + + // join rather than collect the paths, which can cover every file of the table + Dataset<Row> entriesToRewrite = + df.filter(df.col("manifest").isin(manifestsToRewrite.toArray())); + Dataset<Row> markedEntries = markEntriesToRepair(entriesToRewrite, verdicts); + List<ManifestFile> written = writeManifests(content, specId, markedEntries, rewritten); + + return RepairedManifests.of(rewritten, written, repairedCount); + } finally { + verdicts.unpersist(false); + } + }); + } + + private Dataset<Row> markEntriesToRepair(Dataset<Row> entries, Dataset<Row> verdicts) { + Dataset<Row> repairedPaths = + verdicts.select(verdicts.col("path").as("repaired_path")).distinct(); + return entries + .join( + repairedPaths, + entries.col("data_file.file_path").equalTo(repairedPaths.col("repaired_path")), + "left") + .withColumn("repair", functions.col("repaired_path").isNotNull()) + .drop("repaired_path") + .select("manifest", "snapshot_id", "data_file.file_path", "repair"); + } + + private Dataset<Row> buildManifestEntryDF(List<ManifestFile> manifests) { + Dataset<Row> manifestDF = + spark() + .createDataset(Lists.transform(manifests, ManifestFile::path), Encoders.STRING()) + .toDF("manifest"); + + Dataset<Row> entryDF = + loadMetadataTable(table, ENTRIES) + .filter("status < 2") // select only live entries + .selectExpr("input_file_name() as manifest", "snapshot_id", "data_file"); + + return entryDF.join( + manifestDF, manifestDF.col("manifest").equalTo(entryDF.col("manifest")), "left_semi"); + } + + private List<ManifestFile> writeManifests( + ManifestContent content, int specId, Dataset<Row> entryDF, List<ManifestFile> manifests) { + ManifestWriterFactory writers = manifestWriters(specId); + RepairContext context = newRepairContext(content); + + WriteManifests<?> writeFunc = + content == ManifestContent.DATA + ? new WriteDataManifests(writers, manifests, context) + : new WriteDeleteManifests(writers, manifests, context); + + // repartition by manifest so each manifest's entries stay together, keeping the table's layout + return writeFunc + .apply(entryDF.repartition(manifests.size(), entryDF.col("manifest"))) + .collectAsList(); + } + + private RepairContext newRepairContext(ManifestContent content) { + boolean repairColumnMetrics = + PropertyUtil.propertyAsBoolean( + options(), REPAIR_COLUMN_METRICS, REPAIR_COLUMN_METRICS_DEFAULT); + return new RepairContext( + sparkContext().broadcast(SerializableTableWithSize.copyOf(table)), + content, + repairColumnMetrics); + } + + private List<ManifestFile> loadManifests(ManifestContent content, Snapshot snapshot) { + switch (content) { + case DATA: + return snapshot.dataManifests(table.io()); + case DELETES: + return snapshot.deleteManifests(table.io()); + default: + throw new IllegalArgumentException("Unknown manifest content: " + content); + } + } + + private void replaceManifests( + List<ManifestFile> deletedManifests, List<ManifestFile> addedManifests) { + try { + RewriteManifests rewriteManifests = table.rewriteManifests(); + deletedManifests.forEach(rewriteManifests::deleteManifest); + addedManifests.forEach(rewriteManifests::addManifest); + Set<String> deletedPaths = + deletedManifests.stream().map(ManifestFile::path).collect(Collectors.toSet()); + // The validator receives the refreshed parent on every commit attempt, including retries. + // Recompute totals against that parent so concurrent changes are included in the summary. + rewriteManifests.validateWith( + snapshots -> { + Snapshot parent = Iterables.getFirst(snapshots, null); + if (parent == null) { + return false; + } + + List<ManifestFile> manifests = + parent.allManifests(table.io()).stream() + .filter(manifest -> !deletedPaths.contains(manifest.path())) + .collect(Collectors.toList()); + manifests.addAll(addedManifests); + try { + updateSnapshotTotals(rewriteManifests, manifests); + } catch (Exception e) { + // validation runs before the commit is published, so core and the handler below can + // safely delete the staged and replacement manifests + throw new SnapshotTotalsFailure(e); + } + + return true; + }); + commit(rewriteManifests); + + if (shouldStageManifests) { + // delete new manifests as they were rewritten before the commit + deleteFiles(Iterables.transform(addedManifests, ManifestFile::path)); + } + } catch (CommitStateUnknownException e) { + // don't clean up added manifest files, because they may have been successfully committed + throw e; + } catch (Exception e) { + if (e instanceof CleanableFailure) { + deleteFiles(Iterables.transform(addedManifests, ManifestFile::path)); + } + + throw e; + } + } + + private static class SnapshotTotalsFailure extends RuntimeException implements CleanableFailure { + SnapshotTotalsFailure(Exception cause) { + super("Cannot update snapshot totals during repair", cause); + } + } + + private void updateSnapshotTotals(RewriteManifests update, List<ManifestFile> manifests) { + Broadcast<Table> tableBroadcast = + sparkContext().broadcast(SerializableTableWithSize.copyOf(table)); + List<Row> totals = + spark() + .createDataset(manifests, Encoders.javaSerialization(ManifestFile.class)) + .mapPartitions( + new ReadSnapshotTotals(tableBroadcast), + Encoders.tuple(Encoders.STRING(), Encoders.LONG())) + .toDF("metric", "value") + .groupBy("metric") + .sum("value") + .collectAsList(); + totals.forEach(row -> update.set(row.getString(0), Long.toString(row.getLong(1)))); + } + + private static class ReadSnapshotTotals + implements MapPartitionsFunction<ManifestFile, Tuple2<String, Long>> { + private static final List<String> COLUMNS = + ImmutableList.of( + "content", + "file_format", + "record_count", + "file_size_in_bytes", + "content_size_in_bytes"); + + private final Broadcast<Table> tableBroadcast; + + ReadSnapshotTotals(Broadcast<Table> tableBroadcast) { + this.tableBroadcast = tableBroadcast; + } + + @Override + public Iterator<Tuple2<String, Long>> call(Iterator<ManifestFile> manifests) throws Exception { + Table table = tableBroadcast.value(); + Map<String, Long> totals = Maps.newHashMap(); + totals.put(SnapshotSummary.TOTAL_RECORDS_PROP, 0L); + totals.put(SnapshotSummary.TOTAL_FILE_SIZE_PROP, 0L); + totals.put(SnapshotSummary.TOTAL_DATA_FILES_PROP, 0L); + totals.put(SnapshotSummary.TOTAL_DELETE_FILES_PROP, 0L); + totals.put(SnapshotSummary.TOTAL_POS_DELETES_PROP, 0L); + totals.put(SnapshotSummary.TOTAL_EQ_DELETES_PROP, 0L); + while (manifests.hasNext()) { + ManifestFile manifest = manifests.next(); + if (manifest.snapshotId() == null) { + // Replacement entries have explicit lineage, but readers also require a manifest ID. + manifest = GenericManifestFile.copyOf(manifest).withSnapshotId(-1L).build(); + } + + try (ManifestReader<?> reader = + manifest.content() == ManifestContent.DATA + ? ManifestFiles.read(manifest, table.io(), table.specs()) + : ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (ContentFile<?> file : reader.select(COLUMNS)) { + totals.merge( + SnapshotSummary.TOTAL_FILE_SIZE_PROP, + ScanTaskUtil.contentSizeInBytes(file), + Long::sum); + switch (file.content()) { + case DATA -> { + totals.merge(SnapshotSummary.TOTAL_DATA_FILES_PROP, 1L, Long::sum); + totals.merge(SnapshotSummary.TOTAL_RECORDS_PROP, file.recordCount(), Long::sum); + } + case POSITION_DELETES -> { + totals.merge(SnapshotSummary.TOTAL_DELETE_FILES_PROP, 1L, Long::sum); + totals.merge(SnapshotSummary.TOTAL_POS_DELETES_PROP, file.recordCount(), Long::sum); + } + case EQUALITY_DELETES -> { + totals.merge(SnapshotSummary.TOTAL_DELETE_FILES_PROP, 1L, Long::sum); + totals.merge(SnapshotSummary.TOTAL_EQ_DELETES_PROP, file.recordCount(), Long::sum); + } + } + } + } + } + + return totals.entrySet().stream() + .map(entry -> new Tuple2<>(entry.getKey(), entry.getValue())) + .iterator(); + } + } + + private void deleteFiles(Iterable<String> locations) { + Iterable<FileInfo> files = + Iterables.transform(locations, location -> new FileInfo(location, MANIFEST)); + if (table.io() instanceof SupportsBulkOperations) { + deleteFiles((SupportsBulkOperations) table.io(), files.iterator()); + } else { + deleteFiles( + ThreadPools.getWorkerPool(), file -> table.io().deleteFile(file), files.iterator()); + } + } + + private ManifestWriterFactory manifestWriters(int specId) { + return new ManifestWriterFactory( + sparkContext().broadcast(SerializableTableWithSize.copyOf(table)), + formatVersion, + specId, + outputLocation, + // allow the actual size of manifests to be 20% higher as the estimation is not precise + (long) (1.2 * targetManifestSizeBytes)); + } + + private <T, U> U withReusableDS(Dataset<T> ds, Function<Dataset<T>, U> func) { + boolean useCaching = + PropertyUtil.propertyAsBoolean(options(), USE_CACHING, USE_CACHING_DEFAULT); + Dataset<T> reusableDS = useCaching ? ds.cache() : ds; + + try { + return func.apply(reusableDS); + } finally { + if (useCaching) { + reusableDS.unpersist(false); + } + } + } + + /** The outcome of repairing the manifests of one content type. */ + private static class RepairedManifests { + private final List<ManifestFile> repairedManifests; + private final List<ManifestFile> newManifests; + private final long repairedCount; + + private RepairedManifests( + List<ManifestFile> repairedManifests, List<ManifestFile> newManifests, long repairedCount) { + this.repairedManifests = repairedManifests; + this.newManifests = newManifests; + this.repairedCount = repairedCount; + } + + static RepairedManifests empty() { + return new RepairedManifests(ImmutableList.of(), ImmutableList.of(), 0L); + } + + static RepairedManifests of( + List<ManifestFile> repairedManifests, List<ManifestFile> newManifests, long repairedCount) { + return new RepairedManifests(repairedManifests, newManifests, repairedCount); + } + + List<ManifestFile> repairedManifests() { + return repairedManifests; + } + + List<ManifestFile> newManifests() { + return newManifests; + } + + long repairedCount() { + return repairedCount; + } + } + + private static class RepairContext implements Serializable { + private final Broadcast<Table> tableBroadcast; + private final ManifestContent content; + private final boolean repairColumnMetrics; + + private transient EncryptingFileIO lazyEncryptingIO = null; + private transient Map<FileContent, MetricsConfig> lazyMetricsConfigs = null; + private transient NameMapping lazyNameMapping = null; + private transient boolean nameMappingResolved = false; + + RepairContext( + Broadcast<Table> tableBroadcast, ManifestContent content, boolean repairColumnMetrics) { + this.tableBroadcast = tableBroadcast; + this.content = content; + this.repairColumnMetrics = repairColumnMetrics; + } + + Table table() { + return tableBroadcast.value(); + } + + FileIO io() { + return table().io(); + } + + EncryptingFileIO encryptingIO() { + if (lazyEncryptingIO == null) { + this.lazyEncryptingIO = EncryptingFileIO.combine(table().io(), table().encryption()); + } + + return lazyEncryptingIO; + } + + InputFile newInputFile(ContentFile<?> file, long fileSizeInBytes) { + if (file.keyMetadata() != null) { + return encryptingIO() + .newDecryptingInputFile(file.location(), fileSizeInBytes, file.keyMetadata()); + } + + return encryptingIO().newInputFile(file.location(), fileSizeInBytes); + } + + boolean repairColumnMetrics() { + return repairColumnMetrics; + } + + PartitionSpec spec(int id) { + return table().specs().get(id); + } + + // a delete manifest can hold both position and equality deletes, whose configs differ + MetricsConfig metricsConfig(ContentFile<?> file) { + if (lazyMetricsConfigs == null) { + this.lazyMetricsConfigs = new EnumMap<>(FileContent.class); + } + + return lazyMetricsConfigs.computeIfAbsent( + file.content(), fileContent -> RepairMetrics.metricsConfig(table(), fileContent)); + } + + NameMapping nameMapping() { + if (!nameMappingResolved) { + this.lazyNameMapping = RepairMetrics.nameMapping(table()); + this.nameMappingResolved = true; + } + + return lazyNameMapping; + } + + SparkContentFile<?> newFileWrapper(Types.StructType combinedFileType, StructType sparkType) { + return content == ManifestContent.DATA + ? new SparkDataFile(combinedFileType, sparkType) + : new SparkDeleteFile(combinedFileType, sparkType); + } + } + + private static class CheckStats implements MapPartitionsFunction<Row, Tuple2<String, String>> { + private final RepairContext context; + + CheckStats(RepairContext context) { + this.context = context; + } + + @Override + public Iterator<Tuple2<String, String>> call(Iterator<Row> rows) { + List<Tuple2<String, String>> verdicts = Lists.newArrayList(); + Types.StructType combinedFileType = + DataFile.getType(Partitioning.partitionType(context.table())); + SparkContentFile<?> fileWrapper = null; + + while (rows.hasNext()) { + Row row = rows.next(); + String manifest = row.getString(0); + Row fileRow = row.getStruct(2); + if (fileWrapper == null) { + fileWrapper = context.newFileWrapper(combinedFileType, (StructType) fileRow.schema()); + } + + ContentFile<?> file = (ContentFile<?>) fileWrapper.wrap(fileRow); + + if (!RepairMetrics.supportsMetrics(file)) { + continue; + } + + String location = file.location().toString(); + + try { + long fileSizeInBytes = context.io().newInputFile(location).getLength(); + InputFile input = context.newInputFile(file, fileSizeInBytes); + Metrics metrics = + RepairMetrics.readMetrics( + input, + file, + context.metricsConfig(file), + context.nameMapping(), + context.table().schema()); + + if (RepairMetrics.statsAreIncorrect( + file, metrics, fileSizeInBytes, context.repairColumnMetrics())) { + verdicts.add(new Tuple2<>(manifest, location)); + } + } catch (Exception e) { + // A file we can't read keeps its stats as is, so skipping can only leave an entry + // unrepaired, never make it worse. + LOG.warn("Skipping the entry of {} as its statistics could not be read", location, e); + } + } + + return verdicts.iterator(); + } + } + + private static class WriteDataManifests extends WriteManifests<DataFile> { + WriteDataManifests( + ManifestWriterFactory writers, List<ManifestFile> manifests, RepairContext context) { + super(writers, manifests, context); + } + + @Override + protected ManifestReader<DataFile> readManifest(ManifestFile manifest, Table table) { + return ManifestFiles.read(manifest, table.io(), table.specs()); + } + + @Override + protected RollingManifestWriter<DataFile> newManifestWriter() { + return writers().newRollingManifestWriter(); + } + } + + private static class WriteDeleteManifests extends WriteManifests<DeleteFile> { + WriteDeleteManifests( + ManifestWriterFactory writers, List<ManifestFile> manifests, RepairContext context) { + super(writers, manifests, context); + } + + @Override + protected ManifestReader<DeleteFile> readManifest(ManifestFile manifest, Table table) { + return ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs()); + } + + @Override + protected RollingManifestWriter<DeleteFile> newManifestWriter() { + return writers().newRollingDeleteManifestWriter(); + } + } + + // entries are written as existing with their original snapshot id and sequence numbers, so the + // delete files that apply to them still do + private abstract static class WriteManifests<F extends ContentFile<F>> + implements MapPartitionsFunction<Row, ManifestFile> { + + private static final Encoder<ManifestFile> MANIFEST_ENCODER = + Encoders.javaSerialization(ManifestFile.class); + + private final ManifestWriterFactory writers; + private final List<ManifestFile> manifests; + private final RepairContext context; + + WriteManifests( + ManifestWriterFactory writers, List<ManifestFile> manifests, RepairContext context) { + this.writers = writers; + this.manifests = manifests; + this.context = context; + } + + protected abstract ManifestReader<F> readManifest(ManifestFile manifest, Table table); + + protected abstract RollingManifestWriter<F> newManifestWriter(); + + public Dataset<ManifestFile> apply(Dataset<Row> input) { + return input.mapPartitions(this, MANIFEST_ENCODER); + } + + @Override + @SuppressWarnings("unchecked") + public Iterator<ManifestFile> call(Iterator<Row> rows) throws Exception { + Map<String, Map<String, Row>> entriesByManifest = Maps.newHashMap(); + rows.forEachRemaining( + row -> + entriesByManifest + .computeIfAbsent(row.getString(0), path -> Maps.newHashMap()) + .put(row.getString(2), row)); + RollingManifestWriter<F> writer = newManifestWriter(); + + try (writer) { + for (ManifestFile manifest : manifests) { + Map<String, Row> entries = entriesByManifest.get(manifest.path()); + if (entries == null) { + continue; + } + + // read the original manifest, since the entries metadata table omits partition fields + // whose source column was dropped + try (ManifestReader<F> reader = readManifest(manifest, context.table())) { + for (F file : reader) { + Row row = entries.get(file.location()); + Preconditions.checkState( + row != null, "Cannot find entry for %s in %s", file.location(), manifest.path()); + F repaired = row.getBoolean(3) ? (F) repairStats(file) : file; + writer.existing( + repaired, row.getLong(1), file.dataSequenceNumber(), file.fileSequenceNumber()); + } + } + } + } catch (Exception e) { + // a failed task attempt never returns its manifests to the driver, so delete them here + deleteManifests(writer, e); + throw e; + } + + return writer.toManifestFiles().iterator(); + } + + private void deleteManifests(RollingManifestWriter<F> writer, Exception failure) { + try { + Tasks.foreach(writer.toManifestFiles()) + .suppressFailureWhenFinished() + .onFailure((manifest, exc) -> failure.addSuppressed(exc)) + .run(manifest -> context.io().deleteFile(manifest.path())); + } catch (RuntimeException e) { + // the writer did not close, so the manifests it wrote are unknown + failure.addSuppressed(e); + } + } + + /** Rebuilds the file with the statistics read from the file itself. */ + private ContentFile<?> repairStats(ContentFile<?> file) { + long fileSizeInBytes = context.io().newInputFile(file.location()).getLength(); + InputFile input = context.newInputFile(file, fileSizeInBytes); + Metrics metrics = + RepairMetrics.readMetrics( + input, + file, + context.metricsConfig(file), + context.nameMapping(), + context.table().schema()); + if (!context.repairColumnMetrics()) { + // only the record count and file size are being repaired, so keep the stored column stats + metrics = RepairMetrics.recordCountOnly(file, metrics); + } + + return RepairMetrics.withStats(file, context.spec(file.specId()), metrics, fileSizeInBytes); + } + + protected ManifestWriterFactory writers() { + return writers; + } + } + + private static class ManifestWriterFactory implements Serializable { Review Comment: This is nearly identical to the factory in `RewriteManifestsSparkAction`, apart from the manifest file prefix. Can we share one factory and pass the prefix in? Same for the `outputLocation` computation in the constructor. ########## spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RepairTableSparkAction.java: ########## @@ -0,0 +1,905 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.actions; + +import static org.apache.iceberg.MetadataTableType.ENTRIES; + +import java.io.Serializable; +import java.util.EnumMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.hadoop.fs.Path; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.GenericManifestFile; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManifestContent; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.ManifestWriter; +import org.apache.iceberg.Metrics; +import org.apache.iceberg.MetricsConfig; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Partitioning; +import org.apache.iceberg.RewriteManifests; +import org.apache.iceberg.RollingManifestWriter; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotSummary; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.actions.ImmutableRepairTable; +import org.apache.iceberg.actions.RepairTable; +import org.apache.iceberg.encryption.EncryptedOutputFile; +import org.apache.iceberg.encryption.EncryptingFileIO; +import org.apache.iceberg.exceptions.CleanableFailure; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.io.SupportsBulkOperations; +import org.apache.iceberg.mapping.NameMapping; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.spark.JobGroupInfo; +import org.apache.iceberg.spark.SparkContentFile; +import org.apache.iceberg.spark.SparkDataFile; +import org.apache.iceberg.spark.SparkDeleteFile; +import org.apache.iceberg.spark.source.SerializableTableWithSize; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.PropertyUtil; +import org.apache.iceberg.util.ScanTaskUtil; +import org.apache.iceberg.util.Tasks; +import org.apache.iceberg.util.ThreadPools; +import org.apache.spark.api.java.function.MapPartitionsFunction; +import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoder; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.functions; +import org.apache.spark.sql.types.StructType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; + +/** + * An action that repairs incorrect statistics in the manifests of a table, rewriting only the + * manifests that contain an incorrect entry. + * + * <p>Deletion vectors are not repaired. NaN counts and floating-point and geometry bounds are kept + * as stored, because file footers cannot reconstruct them reliably. + */ +public class RepairTableSparkAction extends BaseSnapshotUpdateSparkAction<RepairTableSparkAction> + implements RepairTable { + + public static final String USE_CACHING = "use-caching"; + public static final boolean USE_CACHING_DEFAULT = false; + + /** + * Whether to also repair column statistics, not only record counts and file sizes. Disabled by + * default: the metrics config a file was written under is not recorded, so correct column + * statistics can differ from the recomputed ones after the table's config changes. + */ + public static final String REPAIR_COLUMN_METRICS = "repair-column-metrics"; + + public static final boolean REPAIR_COLUMN_METRICS_DEFAULT = false; + + private static final Logger LOG = LoggerFactory.getLogger(RepairTableSparkAction.class); + + private static final RepairTable.Result EMPTY_RESULT = + ImmutableRepairTable.Result.builder() + .repairedManifests(ImmutableList.of()) + .repairedEntryCount(0L) + .build(); + + private static final String NEW_MANIFEST_PREFIX = "repaired-m-"; + + private final Table table; + private final int formatVersion; + private final long targetManifestSizeBytes; + private final boolean shouldStageManifests; + private final String outputLocation; + + private boolean repairFileMetrics = false; + private boolean dryRun = false; + + RepairTableSparkAction(SparkSession spark, Table table) { + super(spark); + this.table = table; + this.targetManifestSizeBytes = + PropertyUtil.propertyAsLong( + table.properties(), + TableProperties.MANIFEST_TARGET_SIZE_BYTES, + TableProperties.MANIFEST_TARGET_SIZE_BYTES_DEFAULT); + + TableOperations ops = ((HasTableOperations) table).operations(); + Path metadataFilePath = new Path(ops.metadataFileLocation("file")); + this.outputLocation = metadataFilePath.getParent().toString(); + this.formatVersion = ops.current().formatVersion(); + + boolean snapshotIdInheritanceEnabled = + PropertyUtil.propertyAsBoolean( + table.properties(), + TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED, + TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED_DEFAULT); + this.shouldStageManifests = formatVersion == 1 && !snapshotIdInheritanceEnabled; + } + + @Override + protected RepairTableSparkAction self() { + return this; + } + + @Override + public RepairTableSparkAction repairFileMetrics() { + this.repairFileMetrics = true; + return this; + } + + @Override + public RepairTableSparkAction dryRun() { + this.dryRun = true; + return this; + } + + @Override + public RepairTable.Result execute() { + if (!repairFileMetrics) { + return EMPTY_RESULT; + } + + String desc = String.format("Repairing manifests in %s (dryRun=%s)", table.name(), dryRun); + JobGroupInfo info = newJobGroupInfo("REPAIR-TABLE", desc); + return withJobGroupInfo(info, this::doExecute); + } + + private RepairTable.Result doExecute() { + Snapshot currentSnapshot = table.currentSnapshot(); + if (currentSnapshot == null) { + return EMPTY_RESULT; + } + + List<ManifestFile> repairedManifests = Lists.newArrayList(); + List<ManifestFile> newManifests = Lists.newArrayList(); + long repairedCount = 0L; + + try { + for (ManifestContent content : ManifestContent.values()) { + RepairedManifests repaired = repairTable(content, currentSnapshot); + repairedManifests.addAll(repaired.repairedManifests()); + newManifests.addAll(repaired.newManifests()); + repairedCount += repaired.repairedCount(); + } + } catch (Exception e) { + // the manifests of earlier groups are not committed, so delete them rather than orphan them + deleteFiles(Iterables.transform(newManifests, ManifestFile::path)); + throw e; + } + + if (repairedManifests.isEmpty()) { + return EMPTY_RESULT; + } + + if (!dryRun) { + replaceManifests(repairedManifests, newManifests); + } + + LOG.info( + "Repaired the stats of {} manifest entries, rewriting {} manifests as {} (dryRun={})", + repairedCount, + repairedManifests.size(), + newManifests.size(), + dryRun); + + return ImmutableRepairTable.Result.builder() + .repairedManifests(repairedManifests) + .repairedEntryCount(repairedCount) + .build(); + } + + private RepairedManifests repairTable(ManifestContent content, Snapshot snapshot) { + List<ManifestFile> manifests = loadManifests(content, snapshot); + if (manifests.isEmpty()) { + return RepairedManifests.empty(); + } + + // rewriting a manifest with a spec other than its own would change its partition data + Map<Integer, List<ManifestFile>> manifestsBySpecId = + manifests.stream().collect(Collectors.groupingBy(ManifestFile::partitionSpecId)); + + List<ManifestFile> repairedManifests = Lists.newArrayList(); + List<ManifestFile> newManifests = Lists.newArrayList(); + long repairedCount = 0L; + + try { + for (Map.Entry<Integer, List<ManifestFile>> group : manifestsBySpecId.entrySet()) { + RepairedManifests repaired = repairManifests(content, group.getKey(), group.getValue()); + repairedManifests.addAll(repaired.repairedManifests()); + newManifests.addAll(repaired.newManifests()); + repairedCount += repaired.repairedCount(); + } + } catch (Exception e) { + // the manifests of earlier groups are not committed, so delete them rather than orphan them + deleteFiles(Iterables.transform(newManifests, ManifestFile::path)); + throw e; + } + + return RepairedManifests.of(repairedManifests, newManifests, repairedCount); + } + + private RepairedManifests repairManifests( + ManifestContent content, int specId, List<ManifestFile> manifests) { + Dataset<Row> entryDF = buildManifestEntryDF(manifests); + + return withReusableDS( + entryDF, + df -> { + // cached because it is read several times and recomputing it would read every file again + Dataset<Row> verdicts = + df.mapPartitions( + new CheckStats(newRepairContext(content)), + Encoders.tuple(Encoders.STRING(), Encoders.STRING())) + .toDF("manifest", "path") + .cache(); + + try { + Set<String> manifestsToRewrite = + Sets.newHashSet( + verdicts.select("manifest").distinct().as(Encoders.STRING()).collectAsList()); + + if (manifestsToRewrite.isEmpty()) { + return RepairedManifests.empty(); + } + + long repairedCount = verdicts.count(); + List<ManifestFile> rewritten = + manifests.stream() + .filter(manifest -> manifestsToRewrite.contains(manifest.path())) + .collect(Collectors.toList()); + + if (dryRun) { + return RepairedManifests.of(rewritten, ImmutableList.of(), repairedCount); + } + + // join rather than collect the paths, which can cover every file of the table + Dataset<Row> entriesToRewrite = + df.filter(df.col("manifest").isin(manifestsToRewrite.toArray())); + Dataset<Row> markedEntries = markEntriesToRepair(entriesToRewrite, verdicts); + List<ManifestFile> written = writeManifests(content, specId, markedEntries, rewritten); + + return RepairedManifests.of(rewritten, written, repairedCount); + } finally { + verdicts.unpersist(false); + } + }); + } + + private Dataset<Row> markEntriesToRepair(Dataset<Row> entries, Dataset<Row> verdicts) { + Dataset<Row> repairedPaths = + verdicts.select(verdicts.col("path").as("repaired_path")).distinct(); + return entries + .join( + repairedPaths, + entries.col("data_file.file_path").equalTo(repairedPaths.col("repaired_path")), + "left") + .withColumn("repair", functions.col("repaired_path").isNotNull()) + .drop("repaired_path") + .select("manifest", "snapshot_id", "data_file.file_path", "repair"); + } + + private Dataset<Row> buildManifestEntryDF(List<ManifestFile> manifests) { + Dataset<Row> manifestDF = + spark() + .createDataset(Lists.transform(manifests, ManifestFile::path), Encoders.STRING()) + .toDF("manifest"); + + Dataset<Row> entryDF = + loadMetadataTable(table, ENTRIES) + .filter("status < 2") // select only live entries + .selectExpr("input_file_name() as manifest", "snapshot_id", "data_file"); + + return entryDF.join( + manifestDF, manifestDF.col("manifest").equalTo(entryDF.col("manifest")), "left_semi"); + } + + private List<ManifestFile> writeManifests( + ManifestContent content, int specId, Dataset<Row> entryDF, List<ManifestFile> manifests) { + ManifestWriterFactory writers = manifestWriters(specId); + RepairContext context = newRepairContext(content); + + WriteManifests<?> writeFunc = + content == ManifestContent.DATA + ? new WriteDataManifests(writers, manifests, context) + : new WriteDeleteManifests(writers, manifests, context); + + // repartition by manifest so each manifest's entries stay together, keeping the table's layout + return writeFunc + .apply(entryDF.repartition(manifests.size(), entryDF.col("manifest"))) + .collectAsList(); + } + + private RepairContext newRepairContext(ManifestContent content) { + boolean repairColumnMetrics = + PropertyUtil.propertyAsBoolean( + options(), REPAIR_COLUMN_METRICS, REPAIR_COLUMN_METRICS_DEFAULT); + return new RepairContext( + sparkContext().broadcast(SerializableTableWithSize.copyOf(table)), + content, + repairColumnMetrics); + } + + private List<ManifestFile> loadManifests(ManifestContent content, Snapshot snapshot) { + switch (content) { + case DATA: + return snapshot.dataManifests(table.io()); + case DELETES: + return snapshot.deleteManifests(table.io()); + default: + throw new IllegalArgumentException("Unknown manifest content: " + content); + } + } + + private void replaceManifests( + List<ManifestFile> deletedManifests, List<ManifestFile> addedManifests) { + try { + RewriteManifests rewriteManifests = table.rewriteManifests(); + deletedManifests.forEach(rewriteManifests::deleteManifest); + addedManifests.forEach(rewriteManifests::addManifest); + Set<String> deletedPaths = + deletedManifests.stream().map(ManifestFile::path).collect(Collectors.toSet()); + // The validator receives the refreshed parent on every commit attempt, including retries. + // Recompute totals against that parent so concurrent changes are included in the summary. + rewriteManifests.validateWith( + snapshots -> { + Snapshot parent = Iterables.getFirst(snapshots, null); + if (parent == null) { + return false; + } + + List<ManifestFile> manifests = + parent.allManifests(table.io()).stream() + .filter(manifest -> !deletedPaths.contains(manifest.path())) + .collect(Collectors.toList()); + manifests.addAll(addedManifests); + try { + updateSnapshotTotals(rewriteManifests, manifests); + } catch (Exception e) { + // validation runs before the commit is published, so core and the handler below can + // safely delete the staged and replacement manifests + throw new SnapshotTotalsFailure(e); + } + + return true; + }); + commit(rewriteManifests); + + if (shouldStageManifests) { + // delete new manifests as they were rewritten before the commit + deleteFiles(Iterables.transform(addedManifests, ManifestFile::path)); + } + } catch (CommitStateUnknownException e) { + // don't clean up added manifest files, because they may have been successfully committed + throw e; + } catch (Exception e) { + if (e instanceof CleanableFailure) { + deleteFiles(Iterables.transform(addedManifests, ManifestFile::path)); + } + + throw e; + } + } + + private static class SnapshotTotalsFailure extends RuntimeException implements CleanableFailure { + SnapshotTotalsFailure(Exception cause) { + super("Cannot update snapshot totals during repair", cause); + } + } + + private void updateSnapshotTotals(RewriteManifests update, List<ManifestFile> manifests) { + Broadcast<Table> tableBroadcast = + sparkContext().broadcast(SerializableTableWithSize.copyOf(table)); + List<Row> totals = + spark() + .createDataset(manifests, Encoders.javaSerialization(ManifestFile.class)) + .mapPartitions( + new ReadSnapshotTotals(tableBroadcast), + Encoders.tuple(Encoders.STRING(), Encoders.LONG())) + .toDF("metric", "value") + .groupBy("metric") + .sum("value") + .collectAsList(); + totals.forEach(row -> update.set(row.getString(0), Long.toString(row.getLong(1)))); + } + + private static class ReadSnapshotTotals + implements MapPartitionsFunction<ManifestFile, Tuple2<String, Long>> { + private static final List<String> COLUMNS = + ImmutableList.of( + "content", + "file_format", + "record_count", + "file_size_in_bytes", + "content_size_in_bytes"); + + private final Broadcast<Table> tableBroadcast; + + ReadSnapshotTotals(Broadcast<Table> tableBroadcast) { + this.tableBroadcast = tableBroadcast; + } + + @Override + public Iterator<Tuple2<String, Long>> call(Iterator<ManifestFile> manifests) throws Exception { + Table table = tableBroadcast.value(); + Map<String, Long> totals = Maps.newHashMap(); + totals.put(SnapshotSummary.TOTAL_RECORDS_PROP, 0L); + totals.put(SnapshotSummary.TOTAL_FILE_SIZE_PROP, 0L); + totals.put(SnapshotSummary.TOTAL_DATA_FILES_PROP, 0L); + totals.put(SnapshotSummary.TOTAL_DELETE_FILES_PROP, 0L); + totals.put(SnapshotSummary.TOTAL_POS_DELETES_PROP, 0L); + totals.put(SnapshotSummary.TOTAL_EQ_DELETES_PROP, 0L); + while (manifests.hasNext()) { + ManifestFile manifest = manifests.next(); + if (manifest.snapshotId() == null) { + // Replacement entries have explicit lineage, but readers also require a manifest ID. + manifest = GenericManifestFile.copyOf(manifest).withSnapshotId(-1L).build(); + } + + try (ManifestReader<?> reader = + manifest.content() == ManifestContent.DATA + ? ManifestFiles.read(manifest, table.io(), table.specs()) + : ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (ContentFile<?> file : reader.select(COLUMNS)) { + totals.merge( + SnapshotSummary.TOTAL_FILE_SIZE_PROP, + ScanTaskUtil.contentSizeInBytes(file), + Long::sum); + switch (file.content()) { + case DATA -> { + totals.merge(SnapshotSummary.TOTAL_DATA_FILES_PROP, 1L, Long::sum); + totals.merge(SnapshotSummary.TOTAL_RECORDS_PROP, file.recordCount(), Long::sum); + } + case POSITION_DELETES -> { + totals.merge(SnapshotSummary.TOTAL_DELETE_FILES_PROP, 1L, Long::sum); + totals.merge(SnapshotSummary.TOTAL_POS_DELETES_PROP, file.recordCount(), Long::sum); + } + case EQUALITY_DELETES -> { + totals.merge(SnapshotSummary.TOTAL_DELETE_FILES_PROP, 1L, Long::sum); + totals.merge(SnapshotSummary.TOTAL_EQ_DELETES_PROP, file.recordCount(), Long::sum); + } + } + } + } + } + + return totals.entrySet().stream() + .map(entry -> new Tuple2<>(entry.getKey(), entry.getValue())) + .iterator(); + } + } + + private void deleteFiles(Iterable<String> locations) { + Iterable<FileInfo> files = + Iterables.transform(locations, location -> new FileInfo(location, MANIFEST)); + if (table.io() instanceof SupportsBulkOperations) { + deleteFiles((SupportsBulkOperations) table.io(), files.iterator()); + } else { + deleteFiles( + ThreadPools.getWorkerPool(), file -> table.io().deleteFile(file), files.iterator()); + } + } + + private ManifestWriterFactory manifestWriters(int specId) { + return new ManifestWriterFactory( + sparkContext().broadcast(SerializableTableWithSize.copyOf(table)), + formatVersion, + specId, + outputLocation, + // allow the actual size of manifests to be 20% higher as the estimation is not precise + (long) (1.2 * targetManifestSizeBytes)); + } + + private <T, U> U withReusableDS(Dataset<T> ds, Function<Dataset<T>, U> func) { Review Comment: This is the same as the one in `RewriteManifestsSparkAction`. Can we move it to `BaseSparkAction` and share it? ########## spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RepairTableSparkAction.java: ########## @@ -0,0 +1,905 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.actions; + +import static org.apache.iceberg.MetadataTableType.ENTRIES; + +import java.io.Serializable; +import java.util.EnumMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.hadoop.fs.Path; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.GenericManifestFile; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManifestContent; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.ManifestWriter; +import org.apache.iceberg.Metrics; +import org.apache.iceberg.MetricsConfig; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Partitioning; +import org.apache.iceberg.RewriteManifests; +import org.apache.iceberg.RollingManifestWriter; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotSummary; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.actions.ImmutableRepairTable; +import org.apache.iceberg.actions.RepairTable; +import org.apache.iceberg.encryption.EncryptedOutputFile; +import org.apache.iceberg.encryption.EncryptingFileIO; +import org.apache.iceberg.exceptions.CleanableFailure; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.io.SupportsBulkOperations; +import org.apache.iceberg.mapping.NameMapping; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.spark.JobGroupInfo; +import org.apache.iceberg.spark.SparkContentFile; +import org.apache.iceberg.spark.SparkDataFile; +import org.apache.iceberg.spark.SparkDeleteFile; +import org.apache.iceberg.spark.source.SerializableTableWithSize; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.PropertyUtil; +import org.apache.iceberg.util.ScanTaskUtil; +import org.apache.iceberg.util.Tasks; +import org.apache.iceberg.util.ThreadPools; +import org.apache.spark.api.java.function.MapPartitionsFunction; +import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoder; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.functions; +import org.apache.spark.sql.types.StructType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; + +/** + * An action that repairs incorrect statistics in the manifests of a table, rewriting only the + * manifests that contain an incorrect entry. + * + * <p>Deletion vectors are not repaired. NaN counts and floating-point and geometry bounds are kept + * as stored, because file footers cannot reconstruct them reliably. + */ +public class RepairTableSparkAction extends BaseSnapshotUpdateSparkAction<RepairTableSparkAction> + implements RepairTable { + + public static final String USE_CACHING = "use-caching"; + public static final boolean USE_CACHING_DEFAULT = false; + + /** + * Whether to also repair column statistics, not only record counts and file sizes. Disabled by + * default: the metrics config a file was written under is not recorded, so correct column + * statistics can differ from the recomputed ones after the table's config changes. + */ + public static final String REPAIR_COLUMN_METRICS = "repair-column-metrics"; + + public static final boolean REPAIR_COLUMN_METRICS_DEFAULT = false; + + private static final Logger LOG = LoggerFactory.getLogger(RepairTableSparkAction.class); + + private static final RepairTable.Result EMPTY_RESULT = + ImmutableRepairTable.Result.builder() + .repairedManifests(ImmutableList.of()) + .repairedEntryCount(0L) + .build(); + + private static final String NEW_MANIFEST_PREFIX = "repaired-m-"; + + private final Table table; + private final int formatVersion; + private final long targetManifestSizeBytes; + private final boolean shouldStageManifests; + private final String outputLocation; + + private boolean repairFileMetrics = false; + private boolean dryRun = false; + + RepairTableSparkAction(SparkSession spark, Table table) { + super(spark); + this.table = table; + this.targetManifestSizeBytes = + PropertyUtil.propertyAsLong( + table.properties(), + TableProperties.MANIFEST_TARGET_SIZE_BYTES, + TableProperties.MANIFEST_TARGET_SIZE_BYTES_DEFAULT); + + TableOperations ops = ((HasTableOperations) table).operations(); + Path metadataFilePath = new Path(ops.metadataFileLocation("file")); + this.outputLocation = metadataFilePath.getParent().toString(); + this.formatVersion = ops.current().formatVersion(); + + boolean snapshotIdInheritanceEnabled = + PropertyUtil.propertyAsBoolean( + table.properties(), + TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED, + TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED_DEFAULT); + this.shouldStageManifests = formatVersion == 1 && !snapshotIdInheritanceEnabled; + } + + @Override + protected RepairTableSparkAction self() { + return this; + } + + @Override + public RepairTableSparkAction repairFileMetrics() { + this.repairFileMetrics = true; + return this; + } + + @Override + public RepairTableSparkAction dryRun() { + this.dryRun = true; + return this; + } + + @Override + public RepairTable.Result execute() { + if (!repairFileMetrics) { + return EMPTY_RESULT; + } + + String desc = String.format("Repairing manifests in %s (dryRun=%s)", table.name(), dryRun); + JobGroupInfo info = newJobGroupInfo("REPAIR-TABLE", desc); + return withJobGroupInfo(info, this::doExecute); + } + + private RepairTable.Result doExecute() { + Snapshot currentSnapshot = table.currentSnapshot(); + if (currentSnapshot == null) { + return EMPTY_RESULT; + } + + List<ManifestFile> repairedManifests = Lists.newArrayList(); + List<ManifestFile> newManifests = Lists.newArrayList(); + long repairedCount = 0L; + + try { + for (ManifestContent content : ManifestContent.values()) { + RepairedManifests repaired = repairTable(content, currentSnapshot); + repairedManifests.addAll(repaired.repairedManifests()); + newManifests.addAll(repaired.newManifests()); + repairedCount += repaired.repairedCount(); + } + } catch (Exception e) { + // the manifests of earlier groups are not committed, so delete them rather than orphan them + deleteFiles(Iterables.transform(newManifests, ManifestFile::path)); + throw e; + } + + if (repairedManifests.isEmpty()) { + return EMPTY_RESULT; + } + + if (!dryRun) { + replaceManifests(repairedManifests, newManifests); + } + + LOG.info( + "Repaired the stats of {} manifest entries, rewriting {} manifests as {} (dryRun={})", + repairedCount, + repairedManifests.size(), + newManifests.size(), + dryRun); + + return ImmutableRepairTable.Result.builder() + .repairedManifests(repairedManifests) + .repairedEntryCount(repairedCount) + .build(); + } + + private RepairedManifests repairTable(ManifestContent content, Snapshot snapshot) { + List<ManifestFile> manifests = loadManifests(content, snapshot); + if (manifests.isEmpty()) { + return RepairedManifests.empty(); + } + + // rewriting a manifest with a spec other than its own would change its partition data + Map<Integer, List<ManifestFile>> manifestsBySpecId = + manifests.stream().collect(Collectors.groupingBy(ManifestFile::partitionSpecId)); + + List<ManifestFile> repairedManifests = Lists.newArrayList(); + List<ManifestFile> newManifests = Lists.newArrayList(); + long repairedCount = 0L; + + try { + for (Map.Entry<Integer, List<ManifestFile>> group : manifestsBySpecId.entrySet()) { + RepairedManifests repaired = repairManifests(content, group.getKey(), group.getValue()); + repairedManifests.addAll(repaired.repairedManifests()); + newManifests.addAll(repaired.newManifests()); + repairedCount += repaired.repairedCount(); + } + } catch (Exception e) { + // the manifests of earlier groups are not committed, so delete them rather than orphan them + deleteFiles(Iterables.transform(newManifests, ManifestFile::path)); + throw e; + } + + return RepairedManifests.of(repairedManifests, newManifests, repairedCount); + } + + private RepairedManifests repairManifests( + ManifestContent content, int specId, List<ManifestFile> manifests) { + Dataset<Row> entryDF = buildManifestEntryDF(manifests); + + return withReusableDS( + entryDF, + df -> { + // cached because it is read several times and recomputing it would read every file again + Dataset<Row> verdicts = + df.mapPartitions( + new CheckStats(newRepairContext(content)), + Encoders.tuple(Encoders.STRING(), Encoders.STRING())) + .toDF("manifest", "path") + .cache(); + + try { + Set<String> manifestsToRewrite = + Sets.newHashSet( + verdicts.select("manifest").distinct().as(Encoders.STRING()).collectAsList()); + + if (manifestsToRewrite.isEmpty()) { + return RepairedManifests.empty(); + } + + long repairedCount = verdicts.count(); + List<ManifestFile> rewritten = + manifests.stream() + .filter(manifest -> manifestsToRewrite.contains(manifest.path())) + .collect(Collectors.toList()); + + if (dryRun) { + return RepairedManifests.of(rewritten, ImmutableList.of(), repairedCount); + } + + // join rather than collect the paths, which can cover every file of the table + Dataset<Row> entriesToRewrite = + df.filter(df.col("manifest").isin(manifestsToRewrite.toArray())); + Dataset<Row> markedEntries = markEntriesToRepair(entriesToRewrite, verdicts); + List<ManifestFile> written = writeManifests(content, specId, markedEntries, rewritten); + + return RepairedManifests.of(rewritten, written, repairedCount); + } finally { + verdicts.unpersist(false); + } + }); + } + + private Dataset<Row> markEntriesToRepair(Dataset<Row> entries, Dataset<Row> verdicts) { + Dataset<Row> repairedPaths = + verdicts.select(verdicts.col("path").as("repaired_path")).distinct(); + return entries + .join( + repairedPaths, + entries.col("data_file.file_path").equalTo(repairedPaths.col("repaired_path")), + "left") + .withColumn("repair", functions.col("repaired_path").isNotNull()) + .drop("repaired_path") + .select("manifest", "snapshot_id", "data_file.file_path", "repair"); + } + + private Dataset<Row> buildManifestEntryDF(List<ManifestFile> manifests) { + Dataset<Row> manifestDF = + spark() + .createDataset(Lists.transform(manifests, ManifestFile::path), Encoders.STRING()) + .toDF("manifest"); + + Dataset<Row> entryDF = + loadMetadataTable(table, ENTRIES) + .filter("status < 2") // select only live entries + .selectExpr("input_file_name() as manifest", "snapshot_id", "data_file"); + + return entryDF.join( + manifestDF, manifestDF.col("manifest").equalTo(entryDF.col("manifest")), "left_semi"); + } + + private List<ManifestFile> writeManifests( + ManifestContent content, int specId, Dataset<Row> entryDF, List<ManifestFile> manifests) { + ManifestWriterFactory writers = manifestWriters(specId); + RepairContext context = newRepairContext(content); + + WriteManifests<?> writeFunc = + content == ManifestContent.DATA + ? new WriteDataManifests(writers, manifests, context) + : new WriteDeleteManifests(writers, manifests, context); + + // repartition by manifest so each manifest's entries stay together, keeping the table's layout + return writeFunc + .apply(entryDF.repartition(manifests.size(), entryDF.col("manifest"))) + .collectAsList(); + } + + private RepairContext newRepairContext(ManifestContent content) { + boolean repairColumnMetrics = + PropertyUtil.propertyAsBoolean( + options(), REPAIR_COLUMN_METRICS, REPAIR_COLUMN_METRICS_DEFAULT); + return new RepairContext( + sparkContext().broadcast(SerializableTableWithSize.copyOf(table)), + content, + repairColumnMetrics); + } + + private List<ManifestFile> loadManifests(ManifestContent content, Snapshot snapshot) { Review Comment: This can be a switch expression without a default, since the enum is exhaustive. ########## spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RepairTableSparkAction.java: ########## @@ -0,0 +1,905 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.actions; + +import static org.apache.iceberg.MetadataTableType.ENTRIES; + +import java.io.Serializable; +import java.util.EnumMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.hadoop.fs.Path; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.GenericManifestFile; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManifestContent; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.ManifestWriter; +import org.apache.iceberg.Metrics; +import org.apache.iceberg.MetricsConfig; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Partitioning; +import org.apache.iceberg.RewriteManifests; +import org.apache.iceberg.RollingManifestWriter; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotSummary; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.actions.ImmutableRepairTable; +import org.apache.iceberg.actions.RepairTable; +import org.apache.iceberg.encryption.EncryptedOutputFile; +import org.apache.iceberg.encryption.EncryptingFileIO; +import org.apache.iceberg.exceptions.CleanableFailure; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.io.SupportsBulkOperations; +import org.apache.iceberg.mapping.NameMapping; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.spark.JobGroupInfo; +import org.apache.iceberg.spark.SparkContentFile; +import org.apache.iceberg.spark.SparkDataFile; +import org.apache.iceberg.spark.SparkDeleteFile; +import org.apache.iceberg.spark.source.SerializableTableWithSize; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.PropertyUtil; +import org.apache.iceberg.util.ScanTaskUtil; +import org.apache.iceberg.util.Tasks; +import org.apache.iceberg.util.ThreadPools; +import org.apache.spark.api.java.function.MapPartitionsFunction; +import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoder; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.functions; +import org.apache.spark.sql.types.StructType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; + +/** + * An action that repairs incorrect statistics in the manifests of a table, rewriting only the + * manifests that contain an incorrect entry. + * + * <p>Deletion vectors are not repaired. NaN counts and floating-point and geometry bounds are kept + * as stored, because file footers cannot reconstruct them reliably. + */ +public class RepairTableSparkAction extends BaseSnapshotUpdateSparkAction<RepairTableSparkAction> + implements RepairTable { + + public static final String USE_CACHING = "use-caching"; + public static final boolean USE_CACHING_DEFAULT = false; + + /** + * Whether to also repair column statistics, not only record counts and file sizes. Disabled by + * default: the metrics config a file was written under is not recorded, so correct column + * statistics can differ from the recomputed ones after the table's config changes. + */ + public static final String REPAIR_COLUMN_METRICS = "repair-column-metrics"; + + public static final boolean REPAIR_COLUMN_METRICS_DEFAULT = false; + + private static final Logger LOG = LoggerFactory.getLogger(RepairTableSparkAction.class); + + private static final RepairTable.Result EMPTY_RESULT = + ImmutableRepairTable.Result.builder() + .repairedManifests(ImmutableList.of()) + .repairedEntryCount(0L) + .build(); + + private static final String NEW_MANIFEST_PREFIX = "repaired-m-"; + + private final Table table; + private final int formatVersion; + private final long targetManifestSizeBytes; + private final boolean shouldStageManifests; + private final String outputLocation; + + private boolean repairFileMetrics = false; + private boolean dryRun = false; + + RepairTableSparkAction(SparkSession spark, Table table) { + super(spark); + this.table = table; + this.targetManifestSizeBytes = + PropertyUtil.propertyAsLong( + table.properties(), + TableProperties.MANIFEST_TARGET_SIZE_BYTES, + TableProperties.MANIFEST_TARGET_SIZE_BYTES_DEFAULT); + + TableOperations ops = ((HasTableOperations) table).operations(); + Path metadataFilePath = new Path(ops.metadataFileLocation("file")); + this.outputLocation = metadataFilePath.getParent().toString(); + this.formatVersion = ops.current().formatVersion(); + + boolean snapshotIdInheritanceEnabled = + PropertyUtil.propertyAsBoolean( + table.properties(), + TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED, + TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED_DEFAULT); + this.shouldStageManifests = formatVersion == 1 && !snapshotIdInheritanceEnabled; + } + + @Override + protected RepairTableSparkAction self() { + return this; + } + + @Override + public RepairTableSparkAction repairFileMetrics() { + this.repairFileMetrics = true; + return this; + } + + @Override + public RepairTableSparkAction dryRun() { + this.dryRun = true; + return this; + } + + @Override + public RepairTable.Result execute() { + if (!repairFileMetrics) { + return EMPTY_RESULT; + } + + String desc = String.format("Repairing manifests in %s (dryRun=%s)", table.name(), dryRun); + JobGroupInfo info = newJobGroupInfo("REPAIR-TABLE", desc); + return withJobGroupInfo(info, this::doExecute); + } + + private RepairTable.Result doExecute() { + Snapshot currentSnapshot = table.currentSnapshot(); + if (currentSnapshot == null) { + return EMPTY_RESULT; + } + + List<ManifestFile> repairedManifests = Lists.newArrayList(); + List<ManifestFile> newManifests = Lists.newArrayList(); + long repairedCount = 0L; + + try { + for (ManifestContent content : ManifestContent.values()) { + RepairedManifests repaired = repairTable(content, currentSnapshot); + repairedManifests.addAll(repaired.repairedManifests()); + newManifests.addAll(repaired.newManifests()); + repairedCount += repaired.repairedCount(); + } + } catch (Exception e) { + // the manifests of earlier groups are not committed, so delete them rather than orphan them + deleteFiles(Iterables.transform(newManifests, ManifestFile::path)); + throw e; + } + + if (repairedManifests.isEmpty()) { + return EMPTY_RESULT; + } + + if (!dryRun) { + replaceManifests(repairedManifests, newManifests); + } + + LOG.info( + "Repaired the stats of {} manifest entries, rewriting {} manifests as {} (dryRun={})", + repairedCount, + repairedManifests.size(), + newManifests.size(), + dryRun); + + return ImmutableRepairTable.Result.builder() + .repairedManifests(repairedManifests) + .repairedEntryCount(repairedCount) + .build(); + } + + private RepairedManifests repairTable(ManifestContent content, Snapshot snapshot) { + List<ManifestFile> manifests = loadManifests(content, snapshot); + if (manifests.isEmpty()) { + return RepairedManifests.empty(); + } + + // rewriting a manifest with a spec other than its own would change its partition data + Map<Integer, List<ManifestFile>> manifestsBySpecId = + manifests.stream().collect(Collectors.groupingBy(ManifestFile::partitionSpecId)); + + List<ManifestFile> repairedManifests = Lists.newArrayList(); + List<ManifestFile> newManifests = Lists.newArrayList(); + long repairedCount = 0L; + + try { + for (Map.Entry<Integer, List<ManifestFile>> group : manifestsBySpecId.entrySet()) { + RepairedManifests repaired = repairManifests(content, group.getKey(), group.getValue()); + repairedManifests.addAll(repaired.repairedManifests()); + newManifests.addAll(repaired.newManifests()); + repairedCount += repaired.repairedCount(); + } + } catch (Exception e) { Review Comment: The cleanup here and in `doExecute` both delete the manifests written so far, one per level. Can we collect the written manifests in one list and clean up in one place? Right now a reader has to check both catches to convince themselves nothing leaks. ########## spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RepairTableSparkAction.java: ########## @@ -0,0 +1,905 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.actions; + +import static org.apache.iceberg.MetadataTableType.ENTRIES; + +import java.io.Serializable; +import java.util.EnumMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.hadoop.fs.Path; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.GenericManifestFile; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManifestContent; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.ManifestWriter; +import org.apache.iceberg.Metrics; +import org.apache.iceberg.MetricsConfig; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Partitioning; +import org.apache.iceberg.RewriteManifests; +import org.apache.iceberg.RollingManifestWriter; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotSummary; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.actions.ImmutableRepairTable; +import org.apache.iceberg.actions.RepairTable; +import org.apache.iceberg.encryption.EncryptedOutputFile; +import org.apache.iceberg.encryption.EncryptingFileIO; +import org.apache.iceberg.exceptions.CleanableFailure; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.io.SupportsBulkOperations; +import org.apache.iceberg.mapping.NameMapping; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.spark.JobGroupInfo; +import org.apache.iceberg.spark.SparkContentFile; +import org.apache.iceberg.spark.SparkDataFile; +import org.apache.iceberg.spark.SparkDeleteFile; +import org.apache.iceberg.spark.source.SerializableTableWithSize; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.PropertyUtil; +import org.apache.iceberg.util.ScanTaskUtil; +import org.apache.iceberg.util.Tasks; +import org.apache.iceberg.util.ThreadPools; +import org.apache.spark.api.java.function.MapPartitionsFunction; +import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoder; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.functions; +import org.apache.spark.sql.types.StructType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; + +/** + * An action that repairs incorrect statistics in the manifests of a table, rewriting only the + * manifests that contain an incorrect entry. + * + * <p>Deletion vectors are not repaired. NaN counts and floating-point and geometry bounds are kept + * as stored, because file footers cannot reconstruct them reliably. + */ +public class RepairTableSparkAction extends BaseSnapshotUpdateSparkAction<RepairTableSparkAction> + implements RepairTable { + + public static final String USE_CACHING = "use-caching"; + public static final boolean USE_CACHING_DEFAULT = false; + + /** + * Whether to also repair column statistics, not only record counts and file sizes. Disabled by + * default: the metrics config a file was written under is not recorded, so correct column + * statistics can differ from the recomputed ones after the table's config changes. + */ + public static final String REPAIR_COLUMN_METRICS = "repair-column-metrics"; + + public static final boolean REPAIR_COLUMN_METRICS_DEFAULT = false; + + private static final Logger LOG = LoggerFactory.getLogger(RepairTableSparkAction.class); + + private static final RepairTable.Result EMPTY_RESULT = + ImmutableRepairTable.Result.builder() + .repairedManifests(ImmutableList.of()) + .repairedEntryCount(0L) + .build(); + + private static final String NEW_MANIFEST_PREFIX = "repaired-m-"; + + private final Table table; + private final int formatVersion; + private final long targetManifestSizeBytes; + private final boolean shouldStageManifests; + private final String outputLocation; + + private boolean repairFileMetrics = false; + private boolean dryRun = false; + + RepairTableSparkAction(SparkSession spark, Table table) { + super(spark); + this.table = table; + this.targetManifestSizeBytes = + PropertyUtil.propertyAsLong( + table.properties(), + TableProperties.MANIFEST_TARGET_SIZE_BYTES, + TableProperties.MANIFEST_TARGET_SIZE_BYTES_DEFAULT); + + TableOperations ops = ((HasTableOperations) table).operations(); + Path metadataFilePath = new Path(ops.metadataFileLocation("file")); + this.outputLocation = metadataFilePath.getParent().toString(); + this.formatVersion = ops.current().formatVersion(); + + boolean snapshotIdInheritanceEnabled = + PropertyUtil.propertyAsBoolean( + table.properties(), + TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED, + TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED_DEFAULT); + this.shouldStageManifests = formatVersion == 1 && !snapshotIdInheritanceEnabled; + } + + @Override + protected RepairTableSparkAction self() { + return this; + } + + @Override + public RepairTableSparkAction repairFileMetrics() { + this.repairFileMetrics = true; + return this; + } + + @Override + public RepairTableSparkAction dryRun() { + this.dryRun = true; + return this; + } + + @Override + public RepairTable.Result execute() { + if (!repairFileMetrics) { + return EMPTY_RESULT; + } + + String desc = String.format("Repairing manifests in %s (dryRun=%s)", table.name(), dryRun); + JobGroupInfo info = newJobGroupInfo("REPAIR-TABLE", desc); + return withJobGroupInfo(info, this::doExecute); + } + + private RepairTable.Result doExecute() { + Snapshot currentSnapshot = table.currentSnapshot(); + if (currentSnapshot == null) { + return EMPTY_RESULT; + } + + List<ManifestFile> repairedManifests = Lists.newArrayList(); + List<ManifestFile> newManifests = Lists.newArrayList(); + long repairedCount = 0L; + + try { + for (ManifestContent content : ManifestContent.values()) { + RepairedManifests repaired = repairTable(content, currentSnapshot); + repairedManifests.addAll(repaired.repairedManifests()); + newManifests.addAll(repaired.newManifests()); + repairedCount += repaired.repairedCount(); + } + } catch (Exception e) { + // the manifests of earlier groups are not committed, so delete them rather than orphan them + deleteFiles(Iterables.transform(newManifests, ManifestFile::path)); + throw e; + } + + if (repairedManifests.isEmpty()) { + return EMPTY_RESULT; + } + + if (!dryRun) { + replaceManifests(repairedManifests, newManifests); + } + + LOG.info( + "Repaired the stats of {} manifest entries, rewriting {} manifests as {} (dryRun={})", + repairedCount, + repairedManifests.size(), + newManifests.size(), + dryRun); + + return ImmutableRepairTable.Result.builder() + .repairedManifests(repairedManifests) + .repairedEntryCount(repairedCount) + .build(); + } + + private RepairedManifests repairTable(ManifestContent content, Snapshot snapshot) { + List<ManifestFile> manifests = loadManifests(content, snapshot); + if (manifests.isEmpty()) { + return RepairedManifests.empty(); + } + + // rewriting a manifest with a spec other than its own would change its partition data + Map<Integer, List<ManifestFile>> manifestsBySpecId = + manifests.stream().collect(Collectors.groupingBy(ManifestFile::partitionSpecId)); + + List<ManifestFile> repairedManifests = Lists.newArrayList(); + List<ManifestFile> newManifests = Lists.newArrayList(); + long repairedCount = 0L; + + try { + for (Map.Entry<Integer, List<ManifestFile>> group : manifestsBySpecId.entrySet()) { + RepairedManifests repaired = repairManifests(content, group.getKey(), group.getValue()); + repairedManifests.addAll(repaired.repairedManifests()); + newManifests.addAll(repaired.newManifests()); + repairedCount += repaired.repairedCount(); + } + } catch (Exception e) { + // the manifests of earlier groups are not committed, so delete them rather than orphan them + deleteFiles(Iterables.transform(newManifests, ManifestFile::path)); + throw e; + } + + return RepairedManifests.of(repairedManifests, newManifests, repairedCount); + } + + private RepairedManifests repairManifests( + ManifestContent content, int specId, List<ManifestFile> manifests) { + Dataset<Row> entryDF = buildManifestEntryDF(manifests); + + return withReusableDS( + entryDF, + df -> { + // cached because it is read several times and recomputing it would read every file again + Dataset<Row> verdicts = + df.mapPartitions( + new CheckStats(newRepairContext(content)), + Encoders.tuple(Encoders.STRING(), Encoders.STRING())) + .toDF("manifest", "path") + .cache(); + + try { + Set<String> manifestsToRewrite = + Sets.newHashSet( + verdicts.select("manifest").distinct().as(Encoders.STRING()).collectAsList()); + + if (manifestsToRewrite.isEmpty()) { + return RepairedManifests.empty(); + } + + long repairedCount = verdicts.count(); + List<ManifestFile> rewritten = + manifests.stream() + .filter(manifest -> manifestsToRewrite.contains(manifest.path())) + .collect(Collectors.toList()); + + if (dryRun) { + return RepairedManifests.of(rewritten, ImmutableList.of(), repairedCount); + } + + // join rather than collect the paths, which can cover every file of the table + Dataset<Row> entriesToRewrite = + df.filter(df.col("manifest").isin(manifestsToRewrite.toArray())); + Dataset<Row> markedEntries = markEntriesToRepair(entriesToRewrite, verdicts); + List<ManifestFile> written = writeManifests(content, specId, markedEntries, rewritten); + + return RepairedManifests.of(rewritten, written, repairedCount); + } finally { + verdicts.unpersist(false); + } + }); + } + + private Dataset<Row> markEntriesToRepair(Dataset<Row> entries, Dataset<Row> verdicts) { + Dataset<Row> repairedPaths = + verdicts.select(verdicts.col("path").as("repaired_path")).distinct(); + return entries + .join( + repairedPaths, + entries.col("data_file.file_path").equalTo(repairedPaths.col("repaired_path")), + "left") + .withColumn("repair", functions.col("repaired_path").isNotNull()) + .drop("repaired_path") + .select("manifest", "snapshot_id", "data_file.file_path", "repair"); + } + + private Dataset<Row> buildManifestEntryDF(List<ManifestFile> manifests) { + Dataset<Row> manifestDF = + spark() + .createDataset(Lists.transform(manifests, ManifestFile::path), Encoders.STRING()) + .toDF("manifest"); + + Dataset<Row> entryDF = + loadMetadataTable(table, ENTRIES) + .filter("status < 2") // select only live entries + .selectExpr("input_file_name() as manifest", "snapshot_id", "data_file"); + + return entryDF.join( + manifestDF, manifestDF.col("manifest").equalTo(entryDF.col("manifest")), "left_semi"); + } + + private List<ManifestFile> writeManifests( + ManifestContent content, int specId, Dataset<Row> entryDF, List<ManifestFile> manifests) { + ManifestWriterFactory writers = manifestWriters(specId); + RepairContext context = newRepairContext(content); + + WriteManifests<?> writeFunc = + content == ManifestContent.DATA + ? new WriteDataManifests(writers, manifests, context) + : new WriteDeleteManifests(writers, manifests, context); + + // repartition by manifest so each manifest's entries stay together, keeping the table's layout + return writeFunc + .apply(entryDF.repartition(manifests.size(), entryDF.col("manifest"))) + .collectAsList(); + } + + private RepairContext newRepairContext(ManifestContent content) { + boolean repairColumnMetrics = + PropertyUtil.propertyAsBoolean( + options(), REPAIR_COLUMN_METRICS, REPAIR_COLUMN_METRICS_DEFAULT); + return new RepairContext( + sparkContext().broadcast(SerializableTableWithSize.copyOf(table)), + content, + repairColumnMetrics); + } + + private List<ManifestFile> loadManifests(ManifestContent content, Snapshot snapshot) { + switch (content) { + case DATA: + return snapshot.dataManifests(table.io()); + case DELETES: + return snapshot.deleteManifests(table.io()); + default: + throw new IllegalArgumentException("Unknown manifest content: " + content); + } + } + + private void replaceManifests( + List<ManifestFile> deletedManifests, List<ManifestFile> addedManifests) { + try { + RewriteManifests rewriteManifests = table.rewriteManifests(); + deletedManifests.forEach(rewriteManifests::deleteManifest); + addedManifests.forEach(rewriteManifests::addManifest); + Set<String> deletedPaths = + deletedManifests.stream().map(ManifestFile::path).collect(Collectors.toSet()); + // The validator receives the refreshed parent on every commit attempt, including retries. + // Recompute totals against that parent so concurrent changes are included in the summary. + rewriteManifests.validateWith( + snapshots -> { + Snapshot parent = Iterables.getFirst(snapshots, null); + if (parent == null) { + return false; + } + + List<ManifestFile> manifests = + parent.allManifests(table.io()).stream() + .filter(manifest -> !deletedPaths.contains(manifest.path())) + .collect(Collectors.toList()); + manifests.addAll(addedManifests); + try { + updateSnapshotTotals(rewriteManifests, manifests); + } catch (Exception e) { + // validation runs before the commit is published, so core and the handler below can + // safely delete the staged and replacement manifests + throw new SnapshotTotalsFailure(e); + } + + return true; + }); + commit(rewriteManifests); + + if (shouldStageManifests) { + // delete new manifests as they were rewritten before the commit + deleteFiles(Iterables.transform(addedManifests, ManifestFile::path)); + } + } catch (CommitStateUnknownException e) { + // don't clean up added manifest files, because they may have been successfully committed + throw e; + } catch (Exception e) { + if (e instanceof CleanableFailure) { + deleteFiles(Iterables.transform(addedManifests, ManifestFile::path)); + } + + throw e; + } + } + + private static class SnapshotTotalsFailure extends RuntimeException implements CleanableFailure { + SnapshotTotalsFailure(Exception cause) { + super("Cannot update snapshot totals during repair", cause); + } + } + + private void updateSnapshotTotals(RewriteManifests update, List<ManifestFile> manifests) { + Broadcast<Table> tableBroadcast = + sparkContext().broadcast(SerializableTableWithSize.copyOf(table)); + List<Row> totals = + spark() + .createDataset(manifests, Encoders.javaSerialization(ManifestFile.class)) + .mapPartitions( + new ReadSnapshotTotals(tableBroadcast), + Encoders.tuple(Encoders.STRING(), Encoders.LONG())) + .toDF("metric", "value") + .groupBy("metric") + .sum("value") + .collectAsList(); + totals.forEach(row -> update.set(row.getString(0), Long.toString(row.getLong(1)))); + } + + private static class ReadSnapshotTotals + implements MapPartitionsFunction<ManifestFile, Tuple2<String, Long>> { + private static final List<String> COLUMNS = + ImmutableList.of( + "content", + "file_format", + "record_count", + "file_size_in_bytes", + "content_size_in_bytes"); + + private final Broadcast<Table> tableBroadcast; + + ReadSnapshotTotals(Broadcast<Table> tableBroadcast) { + this.tableBroadcast = tableBroadcast; + } + + @Override + public Iterator<Tuple2<String, Long>> call(Iterator<ManifestFile> manifests) throws Exception { + Table table = tableBroadcast.value(); + Map<String, Long> totals = Maps.newHashMap(); + totals.put(SnapshotSummary.TOTAL_RECORDS_PROP, 0L); + totals.put(SnapshotSummary.TOTAL_FILE_SIZE_PROP, 0L); + totals.put(SnapshotSummary.TOTAL_DATA_FILES_PROP, 0L); + totals.put(SnapshotSummary.TOTAL_DELETE_FILES_PROP, 0L); + totals.put(SnapshotSummary.TOTAL_POS_DELETES_PROP, 0L); + totals.put(SnapshotSummary.TOTAL_EQ_DELETES_PROP, 0L); + while (manifests.hasNext()) { + ManifestFile manifest = manifests.next(); + if (manifest.snapshotId() == null) { + // Replacement entries have explicit lineage, but readers also require a manifest ID. + manifest = GenericManifestFile.copyOf(manifest).withSnapshotId(-1L).build(); + } + + try (ManifestReader<?> reader = + manifest.content() == ManifestContent.DATA + ? ManifestFiles.read(manifest, table.io(), table.specs()) + : ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { + for (ContentFile<?> file : reader.select(COLUMNS)) { + totals.merge( + SnapshotSummary.TOTAL_FILE_SIZE_PROP, + ScanTaskUtil.contentSizeInBytes(file), + Long::sum); + switch (file.content()) { + case DATA -> { + totals.merge(SnapshotSummary.TOTAL_DATA_FILES_PROP, 1L, Long::sum); + totals.merge(SnapshotSummary.TOTAL_RECORDS_PROP, file.recordCount(), Long::sum); + } + case POSITION_DELETES -> { + totals.merge(SnapshotSummary.TOTAL_DELETE_FILES_PROP, 1L, Long::sum); + totals.merge(SnapshotSummary.TOTAL_POS_DELETES_PROP, file.recordCount(), Long::sum); + } + case EQUALITY_DELETES -> { + totals.merge(SnapshotSummary.TOTAL_DELETE_FILES_PROP, 1L, Long::sum); + totals.merge(SnapshotSummary.TOTAL_EQ_DELETES_PROP, file.recordCount(), Long::sum); + } + } + } + } + } + + return totals.entrySet().stream() + .map(entry -> new Tuple2<>(entry.getKey(), entry.getValue())) + .iterator(); + } + } + + private void deleteFiles(Iterable<String> locations) { + Iterable<FileInfo> files = + Iterables.transform(locations, location -> new FileInfo(location, MANIFEST)); + if (table.io() instanceof SupportsBulkOperations) { + deleteFiles((SupportsBulkOperations) table.io(), files.iterator()); + } else { + deleteFiles( + ThreadPools.getWorkerPool(), file -> table.io().deleteFile(file), files.iterator()); + } + } + + private ManifestWriterFactory manifestWriters(int specId) { + return new ManifestWriterFactory( + sparkContext().broadcast(SerializableTableWithSize.copyOf(table)), + formatVersion, + specId, + outputLocation, + // allow the actual size of manifests to be 20% higher as the estimation is not precise + (long) (1.2 * targetManifestSizeBytes)); + } + + private <T, U> U withReusableDS(Dataset<T> ds, Function<Dataset<T>, U> func) { + boolean useCaching = + PropertyUtil.propertyAsBoolean(options(), USE_CACHING, USE_CACHING_DEFAULT); + Dataset<T> reusableDS = useCaching ? ds.cache() : ds; + + try { + return func.apply(reusableDS); + } finally { + if (useCaching) { + reusableDS.unpersist(false); + } + } + } + + /** The outcome of repairing the manifests of one content type. */ + private static class RepairedManifests { + private final List<ManifestFile> repairedManifests; + private final List<ManifestFile> newManifests; + private final long repairedCount; + + private RepairedManifests( + List<ManifestFile> repairedManifests, List<ManifestFile> newManifests, long repairedCount) { + this.repairedManifests = repairedManifests; + this.newManifests = newManifests; + this.repairedCount = repairedCount; + } + + static RepairedManifests empty() { + return new RepairedManifests(ImmutableList.of(), ImmutableList.of(), 0L); + } + + static RepairedManifests of( + List<ManifestFile> repairedManifests, List<ManifestFile> newManifests, long repairedCount) { + return new RepairedManifests(repairedManifests, newManifests, repairedCount); + } + + List<ManifestFile> repairedManifests() { + return repairedManifests; + } + + List<ManifestFile> newManifests() { + return newManifests; + } + + long repairedCount() { + return repairedCount; + } + } + + private static class RepairContext implements Serializable { + private final Broadcast<Table> tableBroadcast; + private final ManifestContent content; + private final boolean repairColumnMetrics; + + private transient EncryptingFileIO lazyEncryptingIO = null; + private transient Map<FileContent, MetricsConfig> lazyMetricsConfigs = null; + private transient NameMapping lazyNameMapping = null; + private transient boolean nameMappingResolved = false; + + RepairContext( + Broadcast<Table> tableBroadcast, ManifestContent content, boolean repairColumnMetrics) { + this.tableBroadcast = tableBroadcast; + this.content = content; + this.repairColumnMetrics = repairColumnMetrics; + } + + Table table() { + return tableBroadcast.value(); + } + + FileIO io() { + return table().io(); + } + + EncryptingFileIO encryptingIO() { + if (lazyEncryptingIO == null) { + this.lazyEncryptingIO = EncryptingFileIO.combine(table().io(), table().encryption()); + } + + return lazyEncryptingIO; + } + + InputFile newInputFile(ContentFile<?> file, long fileSizeInBytes) { + if (file.keyMetadata() != null) { + return encryptingIO() + .newDecryptingInputFile(file.location(), fileSizeInBytes, file.keyMetadata()); + } + + return encryptingIO().newInputFile(file.location(), fileSizeInBytes); + } + + boolean repairColumnMetrics() { + return repairColumnMetrics; + } + + PartitionSpec spec(int id) { + return table().specs().get(id); + } + + // a delete manifest can hold both position and equality deletes, whose configs differ + MetricsConfig metricsConfig(ContentFile<?> file) { + if (lazyMetricsConfigs == null) { + this.lazyMetricsConfigs = new EnumMap<>(FileContent.class); + } + + return lazyMetricsConfigs.computeIfAbsent( + file.content(), fileContent -> RepairMetrics.metricsConfig(table(), fileContent)); + } + + NameMapping nameMapping() { + if (!nameMappingResolved) { + this.lazyNameMapping = RepairMetrics.nameMapping(table()); + this.nameMappingResolved = true; + } + + return lazyNameMapping; + } + + SparkContentFile<?> newFileWrapper(Types.StructType combinedFileType, StructType sparkType) { + return content == ManifestContent.DATA + ? new SparkDataFile(combinedFileType, sparkType) + : new SparkDeleteFile(combinedFileType, sparkType); + } + } + + private static class CheckStats implements MapPartitionsFunction<Row, Tuple2<String, String>> { + private final RepairContext context; + + CheckStats(RepairContext context) { + this.context = context; + } + + @Override + public Iterator<Tuple2<String, String>> call(Iterator<Row> rows) { + List<Tuple2<String, String>> verdicts = Lists.newArrayList(); + Types.StructType combinedFileType = + DataFile.getType(Partitioning.partitionType(context.table())); + SparkContentFile<?> fileWrapper = null; + + while (rows.hasNext()) { + Row row = rows.next(); + String manifest = row.getString(0); + Row fileRow = row.getStruct(2); + if (fileWrapper == null) { + fileWrapper = context.newFileWrapper(combinedFileType, (StructType) fileRow.schema()); + } + + ContentFile<?> file = (ContentFile<?>) fileWrapper.wrap(fileRow); + + if (!RepairMetrics.supportsMetrics(file)) { + continue; + } + + String location = file.location().toString(); + + try { + long fileSizeInBytes = context.io().newInputFile(location).getLength(); + InputFile input = context.newInputFile(file, fileSizeInBytes); + Metrics metrics = + RepairMetrics.readMetrics( + input, + file, + context.metricsConfig(file), + context.nameMapping(), + context.table().schema()); + + if (RepairMetrics.statsAreIncorrect( + file, metrics, fileSizeInBytes, context.repairColumnMetrics())) { + verdicts.add(new Tuple2<>(manifest, location)); + } + } catch (Exception e) { + // A file we can't read keeps its stats as is, so skipping can only leave an entry + // unrepaired, never make it worse. + LOG.warn("Skipping the entry of {} as its statistics could not be read", location, e); + } + } + + return verdicts.iterator(); + } + } + + private static class WriteDataManifests extends WriteManifests<DataFile> { + WriteDataManifests( + ManifestWriterFactory writers, List<ManifestFile> manifests, RepairContext context) { + super(writers, manifests, context); + } + + @Override + protected ManifestReader<DataFile> readManifest(ManifestFile manifest, Table table) { + return ManifestFiles.read(manifest, table.io(), table.specs()); + } + + @Override + protected RollingManifestWriter<DataFile> newManifestWriter() { + return writers().newRollingManifestWriter(); + } + } + + private static class WriteDeleteManifests extends WriteManifests<DeleteFile> { + WriteDeleteManifests( + ManifestWriterFactory writers, List<ManifestFile> manifests, RepairContext context) { + super(writers, manifests, context); + } + + @Override + protected ManifestReader<DeleteFile> readManifest(ManifestFile manifest, Table table) { + return ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs()); + } + + @Override + protected RollingManifestWriter<DeleteFile> newManifestWriter() { + return writers().newRollingDeleteManifestWriter(); + } + } + + // entries are written as existing with their original snapshot id and sequence numbers, so the + // delete files that apply to them still do + private abstract static class WriteManifests<F extends ContentFile<F>> + implements MapPartitionsFunction<Row, ManifestFile> { + + private static final Encoder<ManifestFile> MANIFEST_ENCODER = + Encoders.javaSerialization(ManifestFile.class); + + private final ManifestWriterFactory writers; + private final List<ManifestFile> manifests; + private final RepairContext context; + + WriteManifests( + ManifestWriterFactory writers, List<ManifestFile> manifests, RepairContext context) { + this.writers = writers; + this.manifests = manifests; + this.context = context; + } + + protected abstract ManifestReader<F> readManifest(ManifestFile manifest, Table table); + + protected abstract RollingManifestWriter<F> newManifestWriter(); + + public Dataset<ManifestFile> apply(Dataset<Row> input) { + return input.mapPartitions(this, MANIFEST_ENCODER); + } + + @Override + @SuppressWarnings("unchecked") + public Iterator<ManifestFile> call(Iterator<Row> rows) throws Exception { + Map<String, Map<String, Row>> entriesByManifest = Maps.newHashMap(); + rows.forEachRemaining( + row -> Review Comment: These row indices only match the select in `markEntriesToRepair`, which is a few calls away. Can we pull the indices into constants so the contract is visible here? ########## spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RepairMetrics.java: ########## @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.actions; + +import static org.apache.iceberg.TableProperties.DEFAULT_NAME_MAPPING; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.FileMetadata; +import org.apache.iceberg.MetadataColumns; +import org.apache.iceberg.Metrics; +import org.apache.iceberg.MetricsConfig; +import org.apache.iceberg.MetricsUtil; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.avro.Avro; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.mapping.NameMapping; +import org.apache.iceberg.mapping.NameMappingParser; +import org.apache.iceberg.orc.OrcMetrics; +import org.apache.iceberg.parquet.ParquetUtil; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.TypeUtil; + +class RepairMetrics { + + private static final Set<Integer> POSITION_DELETE_FIELD_IDS = + ImmutableSet.of( + MetadataColumns.DELETE_FILE_PATH.fieldId(), MetadataColumns.DELETE_FILE_POS.fieldId()); + + private RepairMetrics() {} + + /** Returns the name mapping of the table, or null if the table does not define one. */ + static NameMapping nameMapping(Table table) { + String mapping = table.properties().get(DEFAULT_NAME_MAPPING); + return mapping != null ? NameMappingParser.fromJson(mapping) : null; + } + + // position delete files always record stats for only the path and position columns + static MetricsConfig metricsConfig(Table table, FileContent content) { + return content == FileContent.POSITION_DELETES + ? MetricsConfig.forPositionDelete() + : MetricsConfig.forTable(table); + } + + // deletion vectors are blobs in a Puffin file, so their stats cannot be read from the file + static boolean supportsMetrics(ContentFile<?> file) { + FileFormat format = file.format(); + return format == FileFormat.PARQUET || format == FileFormat.ORC || format == FileFormat.AVRO; + } + + static Metrics readMetrics( + InputFile input, + ContentFile<?> file, + MetricsConfig config, + NameMapping mapping, + Schema schema) { + Metrics metrics = + switch (file.format()) { + case PARQUET -> ParquetUtil.fileMetrics(input, config, mapping); + case ORC -> OrcMetrics.fromInputFile(input, config, mapping); + case AVRO -> new Metrics(Avro.rowCount(input)); + default -> + throw new UnsupportedOperationException( + "Cannot read metrics of format: " + file.format()); + }; + if (file.format() == FileFormat.AVRO) { + return recordCountOnly(file, metrics); + } + + if (file.content() == FileContent.POSITION_DELETES) { + // Match PositionDeleteWriter: counts are omitted, and bounds are kept only for a single path. + int pathId = MetadataColumns.DELETE_FILE_PATH.fieldId(); + ByteBuffer lowerPath = normalize(metrics.lowerBounds()).get(pathId); + ByteBuffer upperPath = normalize(metrics.upperBounds()).get(pathId); + metrics = + lowerPath != null && lowerPath.equals(upperPath) + ? MetricsUtil.copyWithoutFieldCounts(metrics, POSITION_DELETE_FIELD_IDS) + : MetricsUtil.copyWithoutFieldCountsAndBounds(metrics, POSITION_DELETE_FIELD_IDS); + } + + // Footers cannot recover writer-tracked stats: NaN counts and NaN-safe bounds of float columns + // and bounding boxes of geometry and geography columns. Keep the stored bounds of those columns + // so a correct file is not rewritten with weaker stats. Stored NaN-count IDs also cover columns + // that were dropped. + Set<Integer> preservedBoundIds = Sets.newHashSet(normalize(file.nanValueCounts()).keySet()); + TypeUtil.indexById(schema.asStruct()) + .forEach( + (id, field) -> { + Type.TypeID typeId = field.type().typeId(); + if (typeId == Type.TypeID.FLOAT + || typeId == Type.TypeID.DOUBLE + || typeId == Type.TypeID.GEOMETRY + || typeId == Type.TypeID.GEOGRAPHY) { + preservedBoundIds.add(id); + } + }); + + return new Metrics( + metrics.recordCount(), + metrics.columnSizes(), + metrics.valueCounts(), + metrics.nullValueCounts(), + file.nanValueCounts(), + preserveBounds(file.lowerBounds(), metrics.lowerBounds(), preservedBoundIds), + preserveBounds(file.upperBounds(), metrics.upperBounds(), preservedBoundIds), + file.totalBytes(), + null); + } + + static boolean statsAreIncorrect( + ContentFile<?> file, Metrics metrics, long fileSizeInBytes, boolean compareColumnMetrics) { + if (file.fileSizeInBytes() != fileSizeInBytes) { + return true; + } + + if (metrics.recordCount() != null && file.recordCount() != metrics.recordCount()) { + return true; + } + + if (!compareColumnMetrics) { + return false; + } + + return !countsMatch(file.columnSizes(), metrics.columnSizes()) + || !countsMatch(file.valueCounts(), metrics.valueCounts()) + || !countsMatch(file.nullValueCounts(), metrics.nullValueCounts()) + || !boundsMatch(file.lowerBounds(), metrics.lowerBounds()) + || !boundsMatch(file.upperBounds(), metrics.upperBounds()); + } + + static ContentFile<?> withStats( + ContentFile<?> file, PartitionSpec spec, Metrics metrics, long fileSizeInBytes) { + if (file.content() == FileContent.DATA) { + return DataFiles.builder(spec) + .copy((DataFile) file) + .withMetrics(metrics) + .withFileSizeInBytes(fileSizeInBytes) + .build(); + } + + DeleteFile delete = (DeleteFile) file; + FileMetadata.Builder builder = + FileMetadata.deleteFileBuilder(spec) + .copy(delete) + .withMetrics(metrics) + .withFileSizeInBytes(fileSizeInBytes); + List<Integer> equalityFieldIds = delete.equalityFieldIds(); Review Comment: These two have the same body. Can we keep one generic helper? ########## spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RepairTableSparkAction.java: ########## @@ -0,0 +1,905 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.actions; + +import static org.apache.iceberg.MetadataTableType.ENTRIES; + +import java.io.Serializable; +import java.util.EnumMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.hadoop.fs.Path; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.GenericManifestFile; +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.ManifestContent; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; +import org.apache.iceberg.ManifestReader; +import org.apache.iceberg.ManifestWriter; +import org.apache.iceberg.Metrics; +import org.apache.iceberg.MetricsConfig; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Partitioning; +import org.apache.iceberg.RewriteManifests; +import org.apache.iceberg.RollingManifestWriter; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotSummary; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.actions.ImmutableRepairTable; +import org.apache.iceberg.actions.RepairTable; +import org.apache.iceberg.encryption.EncryptedOutputFile; +import org.apache.iceberg.encryption.EncryptingFileIO; +import org.apache.iceberg.exceptions.CleanableFailure; +import org.apache.iceberg.exceptions.CommitStateUnknownException; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.io.SupportsBulkOperations; +import org.apache.iceberg.mapping.NameMapping; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.spark.JobGroupInfo; +import org.apache.iceberg.spark.SparkContentFile; +import org.apache.iceberg.spark.SparkDataFile; +import org.apache.iceberg.spark.SparkDeleteFile; +import org.apache.iceberg.spark.source.SerializableTableWithSize; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.PropertyUtil; +import org.apache.iceberg.util.ScanTaskUtil; +import org.apache.iceberg.util.Tasks; +import org.apache.iceberg.util.ThreadPools; +import org.apache.spark.api.java.function.MapPartitionsFunction; +import org.apache.spark.broadcast.Broadcast; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Encoder; +import org.apache.spark.sql.Encoders; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.functions; +import org.apache.spark.sql.types.StructType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import scala.Tuple2; + +/** + * An action that repairs incorrect statistics in the manifests of a table, rewriting only the + * manifests that contain an incorrect entry. + * + * <p>Deletion vectors are not repaired. NaN counts and floating-point and geometry bounds are kept + * as stored, because file footers cannot reconstruct them reliably. + */ +public class RepairTableSparkAction extends BaseSnapshotUpdateSparkAction<RepairTableSparkAction> + implements RepairTable { + + public static final String USE_CACHING = "use-caching"; + public static final boolean USE_CACHING_DEFAULT = false; + + /** + * Whether to also repair column statistics, not only record counts and file sizes. Disabled by + * default: the metrics config a file was written under is not recorded, so correct column + * statistics can differ from the recomputed ones after the table's config changes. + */ + public static final String REPAIR_COLUMN_METRICS = "repair-column-metrics"; + + public static final boolean REPAIR_COLUMN_METRICS_DEFAULT = false; + + private static final Logger LOG = LoggerFactory.getLogger(RepairTableSparkAction.class); + + private static final RepairTable.Result EMPTY_RESULT = + ImmutableRepairTable.Result.builder() + .repairedManifests(ImmutableList.of()) + .repairedEntryCount(0L) + .build(); + + private static final String NEW_MANIFEST_PREFIX = "repaired-m-"; + + private final Table table; + private final int formatVersion; + private final long targetManifestSizeBytes; + private final boolean shouldStageManifests; + private final String outputLocation; + + private boolean repairFileMetrics = false; + private boolean dryRun = false; + + RepairTableSparkAction(SparkSession spark, Table table) { + super(spark); + this.table = table; + this.targetManifestSizeBytes = + PropertyUtil.propertyAsLong( + table.properties(), + TableProperties.MANIFEST_TARGET_SIZE_BYTES, + TableProperties.MANIFEST_TARGET_SIZE_BYTES_DEFAULT); + + TableOperations ops = ((HasTableOperations) table).operations(); + Path metadataFilePath = new Path(ops.metadataFileLocation("file")); + this.outputLocation = metadataFilePath.getParent().toString(); + this.formatVersion = ops.current().formatVersion(); + + boolean snapshotIdInheritanceEnabled = + PropertyUtil.propertyAsBoolean( + table.properties(), + TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED, + TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED_DEFAULT); + this.shouldStageManifests = formatVersion == 1 && !snapshotIdInheritanceEnabled; + } + + @Override + protected RepairTableSparkAction self() { + return this; + } + + @Override + public RepairTableSparkAction repairFileMetrics() { + this.repairFileMetrics = true; + return this; + } + + @Override + public RepairTableSparkAction dryRun() { + this.dryRun = true; + return this; + } + + @Override + public RepairTable.Result execute() { + if (!repairFileMetrics) { + return EMPTY_RESULT; + } + + String desc = String.format("Repairing manifests in %s (dryRun=%s)", table.name(), dryRun); + JobGroupInfo info = newJobGroupInfo("REPAIR-TABLE", desc); + return withJobGroupInfo(info, this::doExecute); + } + + private RepairTable.Result doExecute() { + Snapshot currentSnapshot = table.currentSnapshot(); + if (currentSnapshot == null) { + return EMPTY_RESULT; + } + + List<ManifestFile> repairedManifests = Lists.newArrayList(); + List<ManifestFile> newManifests = Lists.newArrayList(); + long repairedCount = 0L; + + try { + for (ManifestContent content : ManifestContent.values()) { + RepairedManifests repaired = repairTable(content, currentSnapshot); + repairedManifests.addAll(repaired.repairedManifests()); + newManifests.addAll(repaired.newManifests()); + repairedCount += repaired.repairedCount(); + } + } catch (Exception e) { + // the manifests of earlier groups are not committed, so delete them rather than orphan them + deleteFiles(Iterables.transform(newManifests, ManifestFile::path)); + throw e; + } + + if (repairedManifests.isEmpty()) { + return EMPTY_RESULT; + } + + if (!dryRun) { + replaceManifests(repairedManifests, newManifests); + } + + LOG.info( + "Repaired the stats of {} manifest entries, rewriting {} manifests as {} (dryRun={})", + repairedCount, + repairedManifests.size(), + newManifests.size(), + dryRun); + + return ImmutableRepairTable.Result.builder() + .repairedManifests(repairedManifests) + .repairedEntryCount(repairedCount) + .build(); + } + + private RepairedManifests repairTable(ManifestContent content, Snapshot snapshot) { + List<ManifestFile> manifests = loadManifests(content, snapshot); + if (manifests.isEmpty()) { + return RepairedManifests.empty(); + } + + // rewriting a manifest with a spec other than its own would change its partition data + Map<Integer, List<ManifestFile>> manifestsBySpecId = + manifests.stream().collect(Collectors.groupingBy(ManifestFile::partitionSpecId)); + + List<ManifestFile> repairedManifests = Lists.newArrayList(); + List<ManifestFile> newManifests = Lists.newArrayList(); + long repairedCount = 0L; + + try { + for (Map.Entry<Integer, List<ManifestFile>> group : manifestsBySpecId.entrySet()) { + RepairedManifests repaired = repairManifests(content, group.getKey(), group.getValue()); + repairedManifests.addAll(repaired.repairedManifests()); + newManifests.addAll(repaired.newManifests()); + repairedCount += repaired.repairedCount(); + } + } catch (Exception e) { + // the manifests of earlier groups are not committed, so delete them rather than orphan them + deleteFiles(Iterables.transform(newManifests, ManifestFile::path)); + throw e; + } + + return RepairedManifests.of(repairedManifests, newManifests, repairedCount); + } + + private RepairedManifests repairManifests( + ManifestContent content, int specId, List<ManifestFile> manifests) { + Dataset<Row> entryDF = buildManifestEntryDF(manifests); + + return withReusableDS( + entryDF, + df -> { + // cached because it is read several times and recomputing it would read every file again + Dataset<Row> verdicts = + df.mapPartitions( + new CheckStats(newRepairContext(content)), + Encoders.tuple(Encoders.STRING(), Encoders.STRING())) + .toDF("manifest", "path") + .cache(); + + try { + Set<String> manifestsToRewrite = + Sets.newHashSet( + verdicts.select("manifest").distinct().as(Encoders.STRING()).collectAsList()); + + if (manifestsToRewrite.isEmpty()) { + return RepairedManifests.empty(); + } + + long repairedCount = verdicts.count(); + List<ManifestFile> rewritten = + manifests.stream() + .filter(manifest -> manifestsToRewrite.contains(manifest.path())) + .collect(Collectors.toList()); + + if (dryRun) { + return RepairedManifests.of(rewritten, ImmutableList.of(), repairedCount); + } + + // join rather than collect the paths, which can cover every file of the table + Dataset<Row> entriesToRewrite = + df.filter(df.col("manifest").isin(manifestsToRewrite.toArray())); + Dataset<Row> markedEntries = markEntriesToRepair(entriesToRewrite, verdicts); + List<ManifestFile> written = writeManifests(content, specId, markedEntries, rewritten); + + return RepairedManifests.of(rewritten, written, repairedCount); + } finally { + verdicts.unpersist(false); + } + }); + } + + private Dataset<Row> markEntriesToRepair(Dataset<Row> entries, Dataset<Row> verdicts) { + Dataset<Row> repairedPaths = + verdicts.select(verdicts.col("path").as("repaired_path")).distinct(); + return entries + .join( + repairedPaths, + entries.col("data_file.file_path").equalTo(repairedPaths.col("repaired_path")), + "left") + .withColumn("repair", functions.col("repaired_path").isNotNull()) + .drop("repaired_path") + .select("manifest", "snapshot_id", "data_file.file_path", "repair"); + } + + private Dataset<Row> buildManifestEntryDF(List<ManifestFile> manifests) { + Dataset<Row> manifestDF = + spark() + .createDataset(Lists.transform(manifests, ManifestFile::path), Encoders.STRING()) + .toDF("manifest"); + + Dataset<Row> entryDF = + loadMetadataTable(table, ENTRIES) + .filter("status < 2") // select only live entries + .selectExpr("input_file_name() as manifest", "snapshot_id", "data_file"); + + return entryDF.join( + manifestDF, manifestDF.col("manifest").equalTo(entryDF.col("manifest")), "left_semi"); + } + + private List<ManifestFile> writeManifests( + ManifestContent content, int specId, Dataset<Row> entryDF, List<ManifestFile> manifests) { + ManifestWriterFactory writers = manifestWriters(specId); + RepairContext context = newRepairContext(content); + + WriteManifests<?> writeFunc = + content == ManifestContent.DATA + ? new WriteDataManifests(writers, manifests, context) + : new WriteDeleteManifests(writers, manifests, context); + + // repartition by manifest so each manifest's entries stay together, keeping the table's layout + return writeFunc + .apply(entryDF.repartition(manifests.size(), entryDF.col("manifest"))) + .collectAsList(); + } + + private RepairContext newRepairContext(ManifestContent content) { Review Comment: Each spec group ends up broadcasting the table twice, once for the check and once for the write. Can we broadcast it once and pass it through? ########## spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RepairMetrics.java: ########## @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.spark.actions; + +import static org.apache.iceberg.TableProperties.DEFAULT_NAME_MAPPING; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.apache.iceberg.ContentFile; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.FileFormat; +import org.apache.iceberg.FileMetadata; +import org.apache.iceberg.MetadataColumns; +import org.apache.iceberg.Metrics; +import org.apache.iceberg.MetricsConfig; +import org.apache.iceberg.MetricsUtil; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.avro.Avro; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.mapping.NameMapping; +import org.apache.iceberg.mapping.NameMappingParser; +import org.apache.iceberg.orc.OrcMetrics; +import org.apache.iceberg.parquet.ParquetUtil; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.TypeUtil; + +class RepairMetrics { + + private static final Set<Integer> POSITION_DELETE_FIELD_IDS = + ImmutableSet.of( + MetadataColumns.DELETE_FILE_PATH.fieldId(), MetadataColumns.DELETE_FILE_POS.fieldId()); + + private RepairMetrics() {} + + /** Returns the name mapping of the table, or null if the table does not define one. */ + static NameMapping nameMapping(Table table) { + String mapping = table.properties().get(DEFAULT_NAME_MAPPING); + return mapping != null ? NameMappingParser.fromJson(mapping) : null; + } + + // position delete files always record stats for only the path and position columns + static MetricsConfig metricsConfig(Table table, FileContent content) { + return content == FileContent.POSITION_DELETES + ? MetricsConfig.forPositionDelete() + : MetricsConfig.forTable(table); + } + + // deletion vectors are blobs in a Puffin file, so their stats cannot be read from the file + static boolean supportsMetrics(ContentFile<?> file) { + FileFormat format = file.format(); + return format == FileFormat.PARQUET || format == FileFormat.ORC || format == FileFormat.AVRO; + } + + static Metrics readMetrics( Review Comment: This does more than read: the Avro, position delete and NaN/geometry branches already merge the stored stats, so what it returns is really the metrics the entry should have. Can we rename it to `expectedMetrics`? It reads well at both call sites, comparing stored against expected. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
