rahulsmahadev commented on code in PR #17622:
URL: https://github.com/apache/iceberg/pull/17622#discussion_r4586800090


##########
spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RepairTableSparkAction.java:
##########
@@ -0,0 +1,689 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark.actions;
+
+import static org.apache.iceberg.MetadataTableType.ENTRIES;
+
+import java.io.Serializable;
+import java.util.EnumMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.ContentFile;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileContent;
+import org.apache.iceberg.HasTableOperations;
+import org.apache.iceberg.ManifestContent;
+import org.apache.iceberg.ManifestFile;
+import org.apache.iceberg.ManifestFiles;
+import org.apache.iceberg.ManifestReader;
+import org.apache.iceberg.Metrics;
+import org.apache.iceberg.MetricsConfig;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Partitioning;
+import org.apache.iceberg.RewriteManifests;
+import org.apache.iceberg.RollingManifestWriter;
+import org.apache.iceberg.Snapshot;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.actions.ImmutableRepairTable;
+import org.apache.iceberg.actions.RepairTable;
+import org.apache.iceberg.encryption.EncryptingFileIO;
+import org.apache.iceberg.exceptions.CleanableFailure;
+import org.apache.iceberg.exceptions.CommitStateUnknownException;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.mapping.NameMapping;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.spark.JobGroupInfo;
+import org.apache.iceberg.spark.SparkContentFile;
+import org.apache.iceberg.spark.SparkDataFile;
+import org.apache.iceberg.spark.SparkDeleteFile;
+import org.apache.iceberg.spark.source.SerializableTableWithSize;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.api.java.function.MapPartitionsFunction;
+import org.apache.spark.broadcast.Broadcast;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Encoder;
+import org.apache.spark.sql.Encoders;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.functions;
+import org.apache.spark.sql.types.StructType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.Tuple2;
+
+/**
+ * An action that repairs incorrect statistics in the manifests of a table, 
rewriting only the
+ * manifests that contain an incorrect entry.
+ *
+ * <p>Deletion vectors are not repaired. NaN counts and floating-point and 
geometry bounds are kept
+ * as stored, because file footers cannot reconstruct them reliably.
+ */
+public class RepairTableSparkAction extends 
BaseSnapshotUpdateSparkAction<RepairTableSparkAction>
+    implements RepairTable {
+
+  public static final String USE_CACHING = USE_CACHING_OPTION;
+  public static final boolean USE_CACHING_DEFAULT = USE_CACHING_BY_DEFAULT;
+
+  /**
+   * Whether to also repair column statistics, not only record counts and file 
sizes. Disabled by
+   * default: the metrics config a file was written under is not recorded, so 
correct column
+   * statistics can differ from the recomputed ones after the table's config 
changes.
+   */
+  public static final String REPAIR_COLUMN_METRICS = "repair-column-metrics";
+
+  public static final boolean REPAIR_COLUMN_METRICS_DEFAULT = false;
+
+  private static final Logger LOG = 
LoggerFactory.getLogger(RepairTableSparkAction.class);
+
+  private static final RepairTable.Result EMPTY_RESULT =
+      ImmutableRepairTable.Result.builder()
+          .repairedManifests(ImmutableList.of())
+          .repairedEntryCount(0L)
+          .build();
+
+  private static final String NEW_MANIFEST_PREFIX = "repaired-m-";
+
+  // positions of the columns that markEntriesToRepair selects, which 
WriteManifests reads
+  private static final int MANIFEST_INDEX = 0;
+  private static final int SNAPSHOT_ID_INDEX = 1;
+  private static final int FILE_PATH_INDEX = 2;
+  private static final int REPAIR_INDEX = 3;
+
+  private final Table table;
+  private final int formatVersion;
+  private final long targetManifestSizeBytes;
+  private final boolean shouldStageManifests;
+  private final String outputLocation;
+
+  private boolean repairFileMetrics = false;
+  private boolean dryRun = false;
+
+  RepairTableSparkAction(SparkSession spark, Table table) {
+    super(spark);
+    this.table = table;
+    this.targetManifestSizeBytes =
+        PropertyUtil.propertyAsLong(
+            table.properties(),
+            TableProperties.MANIFEST_TARGET_SIZE_BYTES,
+            TableProperties.MANIFEST_TARGET_SIZE_BYTES_DEFAULT);
+
+    this.outputLocation = ManifestWriterFactory.defaultOutputLocation(table);
+    TableOperations ops = ((HasTableOperations) table).operations();
+    this.formatVersion = ops.current().formatVersion();
+
+    boolean snapshotIdInheritanceEnabled =
+        PropertyUtil.propertyAsBoolean(
+            table.properties(),
+            TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED,
+            TableProperties.SNAPSHOT_ID_INHERITANCE_ENABLED_DEFAULT);
+    this.shouldStageManifests = formatVersion == 1 && 
!snapshotIdInheritanceEnabled;
+  }
+
+  @Override
+  protected RepairTableSparkAction self() {
+    return this;
+  }
+
+  @Override
+  public RepairTableSparkAction repairFileMetrics() {
+    this.repairFileMetrics = true;
+    return this;
+  }
+
+  @Override
+  public RepairTableSparkAction dryRun() {
+    this.dryRun = true;
+    return this;
+  }
+
+  @Override
+  public RepairTable.Result execute() {
+    if (!repairFileMetrics) {
+      return EMPTY_RESULT;
+    }
+
+    String desc = String.format("Repairing manifests in %s (dryRun=%s)", 
table.name(), dryRun);
+    JobGroupInfo info = newJobGroupInfo("REPAIR-TABLE", desc);
+    return withJobGroupInfo(info, this::doExecute);
+  }
+
+  private RepairTable.Result doExecute() {
+    Snapshot currentSnapshot = table.currentSnapshot();
+    if (currentSnapshot == null) {
+      return EMPTY_RESULT;
+    }
+
+    Broadcast<Table> tableBroadcast =
+        sparkContext().broadcast(SerializableTableWithSize.copyOf(table));
+    List<ManifestFile> repairedManifests = Lists.newArrayList();
+    List<ManifestFile> newManifests = Lists.newArrayList();
+    long repairedCount = 0L;
+
+    try {
+      for (ManifestContent content : ManifestContent.values()) {
+        RepairContext context = newRepairContext(tableBroadcast, content);
+        // rewriting a manifest with a spec other than its own would change 
its partition data
+        Map<Integer, List<ManifestFile>> manifestsBySpecId =
+            loadManifests(content, currentSnapshot).stream()
+                .collect(Collectors.groupingBy(ManifestFile::partitionSpecId));
+        for (Map.Entry<Integer, List<ManifestFile>> group : 
manifestsBySpecId.entrySet()) {
+          RepairedManifests repaired = repairManifests(context, 
group.getKey(), group.getValue());
+          repairedManifests.addAll(repaired.repairedManifests());
+          newManifests.addAll(repaired.newManifests());
+          repairedCount += repaired.repairedCount();
+        }
+      }
+    } catch (Exception e) {
+      // none of the manifests written so far are committed, so delete them 
rather than orphan them
+      CatalogUtil.deleteFiles(
+          table.io(), Iterables.transform(newManifests, ManifestFile::path), 
"manifest");
+      throw e;
+    }
+
+    if (repairedManifests.isEmpty()) {
+      return EMPTY_RESULT;
+    }
+
+    if (!dryRun) {
+      replaceManifests(repairedManifests, newManifests);
+    }
+
+    LOG.info(
+        "Repaired the stats of {} manifest entries, rewriting {} manifests as 
{} (dryRun={})",
+        repairedCount,
+        repairedManifests.size(),
+        newManifests.size(),
+        dryRun);
+
+    return ImmutableRepairTable.Result.builder()
+        .repairedManifests(repairedManifests)
+        .repairedEntryCount(repairedCount)
+        .build();
+  }
+
+  private RepairedManifests repairManifests(
+      RepairContext context, int specId, List<ManifestFile> manifests) {
+    Dataset<Row> entryDF = buildManifestEntryDF(manifests);
+
+    return withReusableDS(
+        entryDF,
+        df -> {
+          // cached because it is read several times and recomputing it would 
read every file again
+          Dataset<Row> verdicts =
+              df.mapPartitions(
+                      new CheckStats(context), 
Encoders.tuple(Encoders.STRING(), Encoders.STRING()))
+                  .toDF("manifest", "path")
+                  .cache();
+
+          try {
+            Set<String> manifestsToRewrite =
+                Sets.newHashSet(
+                    
verdicts.select("manifest").distinct().as(Encoders.STRING()).collectAsList());
+
+            if (manifestsToRewrite.isEmpty()) {
+              return RepairedManifests.empty();
+            }
+
+            long repairedCount = verdicts.count();
+            List<ManifestFile> rewritten =
+                manifests.stream()
+                    .filter(manifest -> 
manifestsToRewrite.contains(manifest.path()))
+                    .collect(Collectors.toList());
+
+            if (dryRun) {
+              return RepairedManifests.of(rewritten, ImmutableList.of(), 
repairedCount);
+            }
+
+            // join rather than collect the paths, which can cover every file 
of the table
+            Dataset<Row> entriesToRewrite =
+                
df.filter(df.col("manifest").isin(manifestsToRewrite.toArray()));
+            Dataset<Row> markedEntries = markEntriesToRepair(entriesToRewrite, 
verdicts);
+            List<ManifestFile> written = writeManifests(context, specId, 
markedEntries, rewritten);
+
+            return RepairedManifests.of(rewritten, written, repairedCount);
+          } finally {
+            verdicts.unpersist(false);
+          }
+        });
+  }
+
+  private Dataset<Row> markEntriesToRepair(Dataset<Row> entries, Dataset<Row> 
verdicts) {
+    Dataset<Row> repairedPaths =
+        verdicts.select(verdicts.col("path").as("repaired_path")).distinct();
+    return entries
+        .join(
+            repairedPaths,
+            
entries.col("data_file.file_path").equalTo(repairedPaths.col("repaired_path")),
+            "left")
+        .withColumn("repair", functions.col("repaired_path").isNotNull())
+        .drop("repaired_path")
+        // WriteManifests reads these columns by position, see MANIFEST_INDEX

Review Comment:
   Done.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to