the-other-tim-brown commented on code in PR #801:
URL: https://github.com/apache/incubator-xtable/pull/801#discussion_r3006285382


##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelConversionTarget.java:
##########
@@ -0,0 +1,483 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+package org.apache.xtable.kernel;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import lombok.Getter;
+import lombok.extern.log4j.Log4j2;
+
+import org.apache.hadoop.conf.Configuration;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import io.delta.kernel.Operation;
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.Transaction;
+import io.delta.kernel.TransactionBuilder;
+import io.delta.kernel.TransactionCommitResult;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.exceptions.TableNotFoundException;
+import io.delta.kernel.hook.PostCommitHook;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.Metadata;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterable;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.conversion.TargetTable;
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.exception.UpdateException;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.metadata.TableSyncMetadata;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.model.storage.TableFormat;
+import org.apache.xtable.spi.sync.ConversionTarget;
+
+/**
+ * Implementation of {@link ConversionTarget} for Delta Lake using the Delta 
Kernel API.
+ *
+ * <p>This implementation uses Delta Kernel (io.delta.kernel) instead of Delta 
Standalone for write
+ * operations, providing better compatibility with cloud storage (S3, GCS, 
Azure Blob Storage, HDFS)
+ * and improved support for Delta Lake 3.x features.
+ *
+ * <p><strong>Initialization:</strong> This class supports two initialization 
patterns:
+ *
+ * <ul>
+ *   <li><strong>Factory/ServiceLoader Pattern:</strong> Use the no-arg 
constructor followed by
+ *       {@link #init(TargetTable, Configuration)}. This is used by {@link
+ *       org.apache.xtable.conversion.ConversionTargetFactory}.
+ *   <li><strong>Direct Testing Pattern:</strong> Use the constructor with 
{@link TargetTable} and
+ *       {@link Engine} parameters for direct instantiation with custom 
dependencies in tests.
+ * </ul>
+ *
+ * <p><strong>Important:</strong> Do not mix initialization patterns. If you 
use the parameterized
+ * constructor, do not call {@link #init(TargetTable, Configuration)} 
afterward, as it will
+ * overwrite the custom Engine.
+ *
+ * <p><strong>Exception Handling:</strong> This implementation only catches 
{@link
+ * io.delta.kernel.exceptions.TableNotFoundException} when checking for table 
existence, allowing
+ * other exceptions (network errors, permission issues, corrupted metadata) to 
propagate rather than
+ * being silently masked. This ensures real errors are visible and fail fast.
+ *
+ * <p><strong>Known Limitations:</strong>
+ *
+ * <ul>
+ *   <li><strong>Commit Tags:</strong> Delta Kernel 4.0.0 does not support 
commit tags in commitInfo
+ *       (e.g., XTABLE_METADATA tags). This affects source-to-target commit 
identifier mapping.
+ *   <li><strong>Schema Evolution:</strong> Schema changes are handled through 
Delta Kernel's
+ *       transaction API, which may have different semantics compared to Delta 
Standalone.
+ *   <li><strong>Internal API Usage:</strong> This implementation casts to 
internal classes
+ *       (SnapshotImpl, TableImpl) to access metadata and commit history, as 
Delta Kernel 4.0.0
+ *       lacks public APIs for these operations. These casts are brittle and 
may break on version
+ *       upgrades. Public API alternatives should be used when available.
+ * </ul>
+ *
+ * <p><strong>Implementation Choice:</strong> Delta Kernel API was chosen over 
Delta Standalone to:
+ *
+ * <ul>
+ *   <li>Support newer Delta Lake features and protocol versions
+ *   <li>Align with the Delta Lake community's direction (Kernel is the 
recommended API)
+ *   <li>Reduce dependency on Spark-specific implementations
+ * </ul>
+ *
+ * @see ConversionTarget
+ * @see io.delta.kernel.Table
+ * @see io.delta.kernel.Transaction
+ */
+@Log4j2
+public class DeltaKernelConversionTarget implements ConversionTarget {
+  private static final String DELTA_LOG_RETENTION_DURATION = 
"delta.logRetentionDuration";
+
+  private DeltaKernelSchemaExtractor schemaExtractor;
+  private DeltaKernelPartitionExtractor partitionExtractor;
+  private DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor;
+
+  private String basePath;
+  private long logRetentionInHours;
+  private DeltaKernelConversionTarget.TransactionState transactionState;
+  private Engine engine;
+
+  /**
+   * No-arg constructor for ServiceLoader instantiation. Must call {@link 
#init(TargetTable,
+   * Configuration)} before use.
+   */
+  public DeltaKernelConversionTarget() {}
+
+  /**
+   * Creates a fully initialized DeltaKernelConversionTarget with custom 
Engine. Typically used in
+   * tests. Do not call {@link #init(TargetTable, Configuration)} after this.
+   *
+   * @param targetTable the target table configuration
+   * @param engine custom Delta Kernel engine instance
+   */
+  public DeltaKernelConversionTarget(TargetTable targetTable, Engine engine) {
+    this(
+        targetTable.getBasePath(),
+        targetTable.getMetadataRetention().toHours(),
+        engine,
+        DeltaKernelSchemaExtractor.getInstance(),
+        DeltaKernelPartitionExtractor.getInstance(),
+        DeltaKernelDataFileUpdatesExtractor.builder()
+            .engine(engine)
+            .basePath(targetTable.getBasePath())
+            // Column statistics are not needed for conversion operations
+            .includeColumnStats(false)
+            .build());
+  }
+
+  @VisibleForTesting
+  DeltaKernelConversionTarget(
+      String tableDataPath,
+      long logRetentionInHours,
+      Engine engine,
+      DeltaKernelSchemaExtractor schemaExtractor,
+      DeltaKernelPartitionExtractor partitionExtractor,
+      DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
+    _init(
+        tableDataPath,
+        logRetentionInHours,
+        engine,
+        schemaExtractor,
+        partitionExtractor,
+        dataKernelFileUpdatesExtractor);
+  }
+
+  /**
+   * Private initialization helper to avoid code duplication between 
constructor and init() paths.
+   */
+  private void _init(
+      String tableDataPath,
+      long logRetentionInHours,
+      Engine engine,
+      DeltaKernelSchemaExtractor schemaExtractor,
+      DeltaKernelPartitionExtractor partitionExtractor,
+      DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
+    this.basePath = tableDataPath;
+    this.schemaExtractor = schemaExtractor;
+    this.partitionExtractor = partitionExtractor;
+    this.dataKernelFileUpdatesExtractor = dataKernelFileUpdatesExtractor;
+    this.engine = engine;
+    this.logRetentionInHours = logRetentionInHours;
+  }
+
+  @Override
+  public void init(TargetTable targetTable, Configuration configuration) {
+    Engine engine = DefaultEngine.create(configuration);
+
+    _init(
+        targetTable.getBasePath(),
+        targetTable.getMetadataRetention().toHours(),
+        engine,
+        DeltaKernelSchemaExtractor.getInstance(),
+        DeltaKernelPartitionExtractor.getInstance(),
+        DeltaKernelDataFileUpdatesExtractor.builder()
+            .engine(engine)
+            .basePath(targetTable.getBasePath())
+            // Column statistics are not needed for conversion operations

Review Comment:
   When we convert from one format to another, we actually do want the 
statistics. This allows query engines to leverage them for their planning 
operations for improved efficiency.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to