vinishjail97 commented on code in PR #801:
URL: https://github.com/apache/incubator-xtable/pull/801#discussion_r3075248664


##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelConversionTarget.java:
##########
@@ -0,0 +1,483 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+package org.apache.xtable.kernel;
+
+import static io.delta.kernel.utils.CloseableIterable.inMemoryIterable;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import lombok.Getter;
+import lombok.extern.log4j.Log4j2;
+
+import org.apache.hadoop.conf.Configuration;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import io.delta.kernel.Operation;
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.Transaction;
+import io.delta.kernel.TransactionBuilder;
+import io.delta.kernel.TransactionCommitResult;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.exceptions.TableNotFoundException;
+import io.delta.kernel.hook.PostCommitHook;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.Metadata;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.actions.SingleAction;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterable;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.conversion.TargetTable;
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.exception.UpdateException;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.metadata.TableSyncMetadata;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.model.storage.TableFormat;
+import org.apache.xtable.spi.sync.ConversionTarget;
+
+/**
+ * Implementation of {@link ConversionTarget} for Delta Lake using the Delta 
Kernel API.
+ *
+ * <p>This implementation uses Delta Kernel (io.delta.kernel) instead of Delta 
Standalone for write
+ * operations, providing better compatibility with cloud storage (S3, GCS, 
Azure Blob Storage, HDFS)
+ * and improved support for Delta Lake 3.x features.
+ *
+ * <p><strong>Initialization:</strong> This class supports two initialization 
patterns:
+ *
+ * <ul>
+ *   <li><strong>Factory/ServiceLoader Pattern:</strong> Use the no-arg 
constructor followed by
+ *       {@link #init(TargetTable, Configuration)}. This is used by {@link
+ *       org.apache.xtable.conversion.ConversionTargetFactory}.
+ *   <li><strong>Direct Testing Pattern:</strong> Use the constructor with 
{@link TargetTable} and
+ *       {@link Engine} parameters for direct instantiation with custom 
dependencies in tests.
+ * </ul>
+ *
+ * <p><strong>Important:</strong> Do not mix initialization patterns. If you 
use the parameterized
+ * constructor, do not call {@link #init(TargetTable, Configuration)} 
afterward, as it will
+ * overwrite the custom Engine.
+ *
+ * <p><strong>Exception Handling:</strong> This implementation only catches 
{@link
+ * io.delta.kernel.exceptions.TableNotFoundException} when checking for table 
existence, allowing
+ * other exceptions (network errors, permission issues, corrupted metadata) to 
propagate rather than
+ * being silently masked. This ensures real errors are visible and fail fast.
+ *
+ * <p><strong>Known Limitations:</strong>
+ *
+ * <ul>
+ *   <li><strong>Commit Tags:</strong> Delta Kernel 4.0.0 does not support 
commit tags in commitInfo
+ *       (e.g., XTABLE_METADATA tags). This affects source-to-target commit 
identifier mapping.
+ *       Tracked in: https://github.com/apache/incubator-xtable/issues/819
+ *   <li><strong>Schema Evolution:</strong> Schema changes are handled through 
Delta Kernel's
+ *       transaction API, which may have different semantics compared to Delta 
Standalone.
+ *   <li><strong>Internal API Usage:</strong> This implementation casts to 
internal classes
+ *       (SnapshotImpl, TableImpl) to access metadata and commit history, as 
Delta Kernel 4.0.0
+ *       lacks public APIs for these operations. These casts are brittle and 
may break on version
+ *       upgrades. Public API alternatives should be used when available.
+ * </ul>
+ *
+ * <p><strong>Implementation Choice:</strong> Delta Kernel API was chosen over 
Delta Standalone to:
+ *
+ * <ul>
+ *   <li>Support newer Delta Lake features and protocol versions
+ *   <li>Align with the Delta Lake community's direction (Kernel is the 
recommended API)
+ *   <li>Reduce dependency on Spark-specific implementations
+ * </ul>
+ *
+ * @see ConversionTarget
+ * @see io.delta.kernel.Table
+ * @see io.delta.kernel.Transaction
+ */
+@Log4j2
+public class DeltaKernelConversionTarget implements ConversionTarget {
+  private static final String DELTA_LOG_RETENTION_DURATION = 
"delta.logRetentionDuration";
+
+  private DeltaKernelSchemaExtractor schemaExtractor;
+  private DeltaKernelPartitionExtractor partitionExtractor;
+  private DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor;
+
+  private String basePath;
+  private long logRetentionInHours;
+  private DeltaKernelConversionTarget.TransactionState transactionState;
+  private Engine engine;
+
+  /**
+   * No-arg constructor for ServiceLoader instantiation. Must call {@link 
#init(TargetTable,
+   * Configuration)} before use.
+   */
+  public DeltaKernelConversionTarget() {}
+
+  /**
+   * Creates a fully initialized DeltaKernelConversionTarget with custom 
Engine. Typically used in
+   * tests. Do not call {@link #init(TargetTable, Configuration)} after this.
+   *
+   * @param targetTable the target table configuration
+   * @param engine custom Delta Kernel engine instance
+   */
+  public DeltaKernelConversionTarget(TargetTable targetTable, Engine engine) {
+    this(
+        targetTable.getBasePath(),
+        targetTable.getMetadataRetention().toHours(),
+        engine,
+        DeltaKernelSchemaExtractor.getInstance(),
+        DeltaKernelPartitionExtractor.getInstance(),
+        DeltaKernelDataFileUpdatesExtractor.builder()
+            .engine(engine)
+            .basePath(targetTable.getBasePath())
+            // Column statistics are not needed for conversion operations
+            .includeColumnStats(false)
+            .build());
+  }
+
+  @VisibleForTesting
+  DeltaKernelConversionTarget(
+      String tableDataPath,
+      long logRetentionInHours,
+      Engine engine,
+      DeltaKernelSchemaExtractor schemaExtractor,
+      DeltaKernelPartitionExtractor partitionExtractor,
+      DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
+    _init(
+        tableDataPath,
+        logRetentionInHours,
+        engine,
+        schemaExtractor,
+        partitionExtractor,
+        dataKernelFileUpdatesExtractor);
+  }
+
+  /**
+   * Private initialization helper to avoid code duplication between 
constructor and init() paths.
+   */
+  private void _init(
+      String tableDataPath,
+      long logRetentionInHours,
+      Engine engine,
+      DeltaKernelSchemaExtractor schemaExtractor,
+      DeltaKernelPartitionExtractor partitionExtractor,
+      DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
+    this.basePath = tableDataPath;
+    this.schemaExtractor = schemaExtractor;
+    this.partitionExtractor = partitionExtractor;
+    this.dataKernelFileUpdatesExtractor = dataKernelFileUpdatesExtractor;
+    this.engine = engine;
+    this.logRetentionInHours = logRetentionInHours;
+  }
+
+  @Override
+  public void init(TargetTable targetTable, Configuration configuration) {
+    Engine engine = DefaultEngine.create(configuration);
+
+    _init(
+        targetTable.getBasePath(),
+        targetTable.getMetadataRetention().toHours(),
+        engine,
+        DeltaKernelSchemaExtractor.getInstance(),
+        DeltaKernelPartitionExtractor.getInstance(),
+        DeltaKernelDataFileUpdatesExtractor.builder()
+            .engine(engine)
+            .basePath(targetTable.getBasePath())
+            .includeColumnStats(true)
+            .build());
+  }
+
+  @Override
+  public void beginSync(InternalTable table) {
+    this.transactionState =
+        new DeltaKernelConversionTarget.TransactionState(engine, 
logRetentionInHours);
+  }
+
+  @Override
+  public void syncSchema(InternalSchema schema) {
+    transactionState.setLatestSchema(schema);
+  }
+
+  @Override
+  public void syncPartitionSpec(List<InternalPartitionField> partitionSpec) {
+    if (partitionSpec != null) {
+      Map<String, StructField> spec =
+          partitionExtractor.convertToDeltaPartitionFormat(partitionSpec);
+      for (Map.Entry<String, StructField> partitionEntry : spec.entrySet()) {
+        String partitionColumnName = partitionEntry.getKey();
+        StructField partitionField = partitionEntry.getValue();
+
+        transactionState.addPartitionColumn(partitionColumnName);
+        if (partitionField != null
+            && transactionState.getLatestSchema().fields().stream()
+                .noneMatch(field -> 
field.getName().equals(partitionField.getName()))) {
+          // add generated columns to schema.
+          transactionState.addColumn(partitionField);
+        }
+      }
+    }
+  }
+
+  @Override
+  public void syncMetadata(TableSyncMetadata metadata) {
+    transactionState.setMetadata(metadata);
+  }
+
+  @Override
+  public void syncFilesForSnapshot(List<PartitionFileGroup> 
partitionedDataFiles) {
+    Table table = Table.forPath(engine, basePath);
+    transactionState.setActions(
+        dataKernelFileUpdatesExtractor.applySnapshot(
+            table, partitionedDataFiles, 
transactionState.getLatestSchemaInternal()));
+  }
+
+  @Override
+  public void syncFilesForDiff(InternalFilesDiff internalFilesDiff) {
+    Table table = Table.forPath(engine, basePath);
+    transactionState.setActions(
+        dataKernelFileUpdatesExtractor.applyDiff(
+            internalFilesDiff, basePath, 
table.getLatestSnapshot(engine).getSchema()));
+  }
+
+  @Override
+  public void completeSync() {
+    transactionState.commitTransaction();
+    transactionState = null;
+  }
+
+  @Override
+  public Optional<TableSyncMetadata> getTableMetadata() {
+    Table table = Table.forPath(engine, basePath);
+    Snapshot snapshot = table.getLatestSnapshot(engine);
+
+    // WORKAROUND: Cast to SnapshotImpl (internal class) to access metadata 
configuration.
+    // Delta Kernel 4.0.0 does not provide a public API to access table 
metadata/configuration.
+    // This cast is brittle and may break on Kernel version upgrades.
+    Metadata metadata = ((SnapshotImpl) snapshot).getMetadata();
+
+    // Get configuration from metadata
+    Map<String, String> configuration = metadata.getConfiguration();
+    String metadataJson = configuration.get(TableSyncMetadata.XTABLE_METADATA);
+
+    return TableSyncMetadata.fromJson(metadataJson);
+  }
+
+  @Override
+  public String getTableFormat() {
+    return TableFormat.DELTA;
+  }
+
+  @Override
+  public Optional<String> getTargetCommitIdentifier(String sourceIdentifier) {
+    // Delta Kernel 4.0.0 does not support commit tags in commitInfo, which 
are required for
+    // source-to-target commit identifier mapping. This limitation is 
documented in:
+    // https://github.com/delta-io/delta/issues/6167
+    // XTable tracking issue: 
https://github.com/apache/incubator-xtable/issues/819
+    //
+    // Unlike DeltaConversionTarget (which uses Delta Standalone with commit 
tag support),
+    // DeltaKernelConversionTarget cannot retrieve commit tags from Delta 
Kernel's API.
+    // Rather than silently scanning all commits (O(n) performance cost) and 
always returning
+    // empty, we explicitly throw an exception to indicate this feature is 
unsupported.
+    //
+    // When Delta Kernel adds commit tag support, this method can be 
reimplemented to:
+    // 1. Scan commit history using tableImpl.getChanges(engine, 0, 
currentVersion, actionSet)
+    // 2. Extract tags from CommitInfo.tags MapValue
+    // 3. Parse XTABLE_METADATA from tags and match sourceIdentifier
+    throw new NotSupportedException(
+        "Source-to-target commit identifier mapping is not supported in 
DeltaKernelConversionTarget. "
+            + "Delta Kernel 4.0.0 does not support commit tags in commitInfo. "
+            + "See: https://github.com/delta-io/delta/issues/6167";);
+  }
+
+  private class TransactionState {
+    private final Engine engine;
+    private final long retentionInHours;
+    private final List<String> partitionColumns;
+    @Getter private StructType latestSchema;
+    @Getter private InternalSchema latestSchemaInternal;
+    private TableSyncMetadata metadata;
+    private List<RowBackedAction> actions;
+
+    private TransactionState(Engine engine, long retentionInHours) {
+      this.engine = engine;
+      this.partitionColumns = new ArrayList<>();
+      this.retentionInHours = retentionInHours;
+
+      try {
+        Table table = Table.forPath(engine, basePath);
+        this.latestSchema = table.getLatestSnapshot(engine).getSchema();
+      } catch (TableNotFoundException e) {
+        // Expected: table doesn't exist yet on first sync
+        this.latestSchema = null;
+      }
+      // Let other exceptions propagate (network issues, permissions, 
corrupted metadata, etc.)
+    }
+
+    /**
+     * Adds a partition column name to the list. Package-private to allow 
access from outer class.
+     */
+    void addPartitionColumn(String columnName) {
+      partitionColumns.add(columnName);
+    }
+
+    void setMetadata(TableSyncMetadata metadata) {
+      this.metadata = metadata;
+    }
+
+    /**
+     * Sets the actions to be committed. Converts from Scala Seq to Java List 
for internal storage.
+     */
+    void setActions(Seq<RowBackedAction> scalaActions) {
+      this.actions = JavaConverters.seqAsJavaList(scalaActions);
+    }
+
+    private void addColumn(StructField field) {
+      latestSchema = latestSchema.add(field);
+      latestSchemaInternal = schemaExtractor.toInternalSchema(latestSchema);
+    }
+
+    private void setLatestSchema(InternalSchema schema) {
+      this.latestSchemaInternal = schema;
+      this.latestSchema = schemaExtractor.fromInternalSchema(schema);
+    }
+
+    private void commitTransaction() {
+      boolean tableExists = checkTableExists();
+
+      Operation operation = tableExists ? Operation.WRITE : 
Operation.CREATE_TABLE;
+
+      if (!tableExists) {
+        File tableDir = new File(basePath);
+        if (!tableDir.exists()) {
+          tableDir.mkdirs();
+        }
+      }
+
+      Table table = Table.forPath(engine, basePath);
+      TransactionBuilder txnBuilder =
+          table.createTransactionBuilder(engine, "XTable Delta Sync", 
operation);
+
+      // LIMITATION: Schema evolution for existing tables is NOT supported in 
Delta Kernel 4.0.0.
+      // The withSchema() method only works during CREATE_TABLE operations. 
For existing tables:
+      // - AddFile/RemoveFile actions are created using the old schema from 
existing snapshot
+      // - If source schema has evolved (columns added/removed/type changed), 
the Delta table
+      //   will have mismatched metadata and data, causing query failures or 
incorrect results
+      // This is a known Delta Kernel limitation: 
https://github.com/delta-io/delta/issues/4305
+      if (!tableExists) {
+        txnBuilder = txnBuilder.withSchema(engine, latestSchema);
+
+        if (!partitionColumns.isEmpty()) {
+          txnBuilder = txnBuilder.withPartitionColumns(engine, 
partitionColumns);
+        }
+      }
+
+      Map<String, String> tableProperties = getConfigurationsForDeltaSync();
+      txnBuilder = txnBuilder.withTableProperties(engine, tableProperties);
+
+      Transaction txn = txnBuilder.build(engine);
+      List<Row> allActionRows = new ArrayList<>();
+
+      // Iterate through actions (Java List) and convert to Row format
+      for (RowBackedAction action : actions) {
+
+        if (action instanceof AddFile) {
+          AddFile addFile = (AddFile) action;
+          Row wrappedRow = 
SingleAction.createAddFileSingleAction(addFile.toRow());
+          allActionRows.add(wrappedRow);
+        } else if (action instanceof RemoveFile) {
+          RemoveFile removeFile = (RemoveFile) action;
+          Row wrappedRow = 
SingleAction.createRemoveFileSingleAction(removeFile.toRow());
+          allActionRows.add(wrappedRow);
+        }
+      }
+
+      CloseableIterator<Row> allActionsIterator =
+          new CloseableIterator<Row>() {
+            private int currentIndex = 0;
+
+            @Override
+            public boolean hasNext() {
+              return currentIndex < allActionRows.size();
+            }
+
+            @Override
+            public Row next() {
+              return allActionRows.get(currentIndex++);
+            }
+
+            @Override
+            public void close() {}
+          };
+
+      CloseableIterable<Row> dataActions = 
inMemoryIterable(allActionsIterator);
+
+      try {
+        TransactionCommitResult result = txn.commit(engine, dataActions);
+
+        // Execute PostCommitHooks to create checkpoints and _last_checkpoint 
metadata file
+        List<PostCommitHook> hooks = result.getPostCommitHooks();
+        if (hooks != null && !hooks.isEmpty()) {
+          for (PostCommitHook hook : hooks) {
+            try {
+              hook.threadSafeInvoke(engine);
+            } catch (Exception hookEx) {
+              // Post-commit hooks are optimizations; log but don't fail the 
transaction
+              log.warn("Post-commit hook failed but transaction succeeded", 
hookEx);
+            }
+          }
+        }
+      } catch (Exception e) {
+        throw new UpdateException("Failed to commit Delta Kernel transaction", 
e);
+      }
+
+      // NOTE: Delta Kernel API limitations compared to Delta Standalone:
+      // - Commit tags (like XTABLE_METADATA in commitInfo.tags) are not yet 
supported
+      // - Operation type metadata (like DeltaOperations.Update) is simplified 
to
+      // Operation.WRITE/CREATE_TABLE
+      // - The commit timestamp is managed by Delta Kernel automatically
+    }
+
+    private boolean checkTableExists() {
+      return DeltaKernelUtils.tableExists(engine, basePath);
+    }
+
+    private Map<String, String> getConfigurationsForDeltaSync() {

Review Comment:
   `getConfigurationsForDeltaSync()` does not set 
`minReaderVersion`/`minWriterVersion`, unlike the existing 
`DeltaConversionTarget`. Protocol versions default to Delta Kernel defaults, 
which may not match features used (e.g., generated columns require writer 
version 4). Is this intentional?



##########
xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelDataFileUpdatesExtractor.java:
##########
@@ -0,0 +1,424 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+package org.apache.xtable.kernel;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import scala.collection.JavaConverters;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.util.VectorUtils;
+import io.delta.kernel.types.IntegerType;
+import io.delta.kernel.types.StringType;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.model.schema.PartitionTransformType;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+
+public class TestDeltaKernelDataFileUpdatesExtractor {
+
+  @TempDir private Path tempDir;
+
+  private Engine engine;
+  private DeltaKernelDataFileUpdatesExtractor extractor;
+  private InternalSchema testSchema;
+  private StructType physicalSchema;
+
+  @BeforeEach
+  public void setup() {
+    Configuration hadoopConf = new Configuration();
+    engine = DefaultEngine.create(hadoopConf);
+
+    // Create test schema
+    testSchema =
+        InternalSchema.builder()
+            .name("record")
+            .dataType(InternalType.RECORD)
+            .fields(
+                Arrays.asList(
+                    InternalField.builder()
+                        .name("id")
+                        .schema(
+                            InternalSchema.builder()
+                                .name("integer")
+                                .dataType(InternalType.INT)
+                                .isNullable(false)
+                                .build())
+                        .build(),
+                    InternalField.builder()
+                        .name("name")
+                        .schema(
+                            InternalSchema.builder()
+                                .name("string")
+                                .dataType(InternalType.STRING)
+                                .isNullable(true)
+                                .build())
+                        .build()))
+            .build();
+
+    // Create physical schema
+    physicalSchema =
+        new StructType()
+            .add(new StructField("id", IntegerType.INTEGER, false))
+            .add(new StructField("name", StringType.STRING, true));
+
+    // Initialize extractor
+    extractor =
+        DeltaKernelDataFileUpdatesExtractor.builder()
+            .engine(engine)
+            .basePath(tempDir.toString())
+            .includeColumnStats(false)
+            .build();
+  }
+
+  @Test
+  public void testCreateAddFileAction() throws IOException {
+    // Create a test data file
+    String testFilePath = tempDir.resolve("test_data.parquet").toString();
+    Files.createFile(Paths.get(testFilePath));
+
+    long expectedFileSize = 1024L;
+    long expectedRecordCount = 100L;
+
+    InternalDataFile dataFile =
+        InternalDataFile.builder()
+            .physicalPath(testFilePath)
+            .fileSizeBytes(expectedFileSize)
+            .lastModified(Instant.now().toEpochMilli())
+            .recordCount(expectedRecordCount)
+            .partitionValues(Collections.emptyList())
+            .columnStats(Collections.emptyList())
+            .build();
+
+    // Create a simple Delta table for testing
+    Table table = createSimpleDeltaTable();
+
+    List<PartitionFileGroup> partitionedDataFiles =
+        Collections.singletonList(
+            PartitionFileGroup.builder()
+                .files(Collections.singletonList(dataFile))
+                .partitionValues(Collections.emptyList())
+                .build());
+
+    // Execute applySnapshot
+    scala.collection.Seq<RowBackedAction> actions =
+        extractor.applySnapshot(table, partitionedDataFiles, testSchema);
+
+    // Verify actions are created
+    assertNotNull(actions);
+    List<RowBackedAction> actionList = JavaConverters.seqAsJavaList(actions);
+    assertFalse(actionList.isEmpty(), "Should have at least one action");
+
+    // Verify we have AddFile actions
+    boolean hasAddFile = actionList.stream().anyMatch(action -> action 
instanceof AddFile);
+    assertTrue(hasAddFile, "Should contain AddFile actions");
+
+    // Verify AddFile content
+    AddFile addFile =
+        actionList.stream()
+            .filter(action -> action instanceof AddFile)
+            .map(action -> (AddFile) action)
+            .findFirst()
+            .orElseThrow(() -> new AssertionError("Expected AddFile action not 
found"));
+
+    // Assert on AddFile properties
+    assertTrue(
+        addFile.getPath().contains("test_data.parquet"),
+        "AddFile path should contain the test file name");
+    assertTrue(addFile.getSize() > 0, "AddFile size should be greater than 0");

Review Comment:
   nit: This asserts `addFile.getSize() > 0` but the test creates 
`expectedFileSize = 1024L` (line 128). Use `assertEquals(1024L, 
addFile.getSize())` to validate size propagation — otherwise a wrong size still 
passes.



##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelConversionTarget.java:
##########
@@ -0,0 +1,483 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+package org.apache.xtable.kernel;
+
+import static io.delta.kernel.utils.CloseableIterable.inMemoryIterable;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import lombok.Getter;
+import lombok.extern.log4j.Log4j2;
+
+import org.apache.hadoop.conf.Configuration;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import io.delta.kernel.Operation;
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.Transaction;
+import io.delta.kernel.TransactionBuilder;
+import io.delta.kernel.TransactionCommitResult;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.exceptions.TableNotFoundException;
+import io.delta.kernel.hook.PostCommitHook;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.Metadata;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.actions.SingleAction;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterable;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.conversion.TargetTable;
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.exception.UpdateException;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.metadata.TableSyncMetadata;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.model.storage.TableFormat;
+import org.apache.xtable.spi.sync.ConversionTarget;
+
+/**
+ * Implementation of {@link ConversionTarget} for Delta Lake using the Delta 
Kernel API.
+ *
+ * <p>This implementation uses Delta Kernel (io.delta.kernel) instead of Delta 
Standalone for write
+ * operations, providing better compatibility with cloud storage (S3, GCS, 
Azure Blob Storage, HDFS)
+ * and improved support for Delta Lake 3.x features.
+ *
+ * <p><strong>Initialization:</strong> This class supports two initialization 
patterns:
+ *
+ * <ul>
+ *   <li><strong>Factory/ServiceLoader Pattern:</strong> Use the no-arg 
constructor followed by
+ *       {@link #init(TargetTable, Configuration)}. This is used by {@link
+ *       org.apache.xtable.conversion.ConversionTargetFactory}.
+ *   <li><strong>Direct Testing Pattern:</strong> Use the constructor with 
{@link TargetTable} and
+ *       {@link Engine} parameters for direct instantiation with custom 
dependencies in tests.
+ * </ul>
+ *
+ * <p><strong>Important:</strong> Do not mix initialization patterns. If you 
use the parameterized
+ * constructor, do not call {@link #init(TargetTable, Configuration)} 
afterward, as it will
+ * overwrite the custom Engine.
+ *
+ * <p><strong>Exception Handling:</strong> This implementation only catches 
{@link
+ * io.delta.kernel.exceptions.TableNotFoundException} when checking for table 
existence, allowing
+ * other exceptions (network errors, permission issues, corrupted metadata) to 
propagate rather than
+ * being silently masked. This ensures real errors are visible and fail fast.
+ *
+ * <p><strong>Known Limitations:</strong>
+ *
+ * <ul>
+ *   <li><strong>Commit Tags:</strong> Delta Kernel 4.0.0 does not support 
commit tags in commitInfo
+ *       (e.g., XTABLE_METADATA tags). This affects source-to-target commit 
identifier mapping.
+ *       Tracked in: https://github.com/apache/incubator-xtable/issues/819
+ *   <li><strong>Schema Evolution:</strong> Schema changes are handled through 
Delta Kernel's
+ *       transaction API, which may have different semantics compared to Delta 
Standalone.
+ *   <li><strong>Internal API Usage:</strong> This implementation casts to 
internal classes
+ *       (SnapshotImpl, TableImpl) to access metadata and commit history, as 
Delta Kernel 4.0.0
+ *       lacks public APIs for these operations. These casts are brittle and 
may break on version
+ *       upgrades. Public API alternatives should be used when available.
+ * </ul>
+ *
+ * <p><strong>Implementation Choice:</strong> Delta Kernel API was chosen over 
Delta Standalone to:
+ *
+ * <ul>
+ *   <li>Support newer Delta Lake features and protocol versions
+ *   <li>Align with the Delta Lake community's direction (Kernel is the 
recommended API)
+ *   <li>Reduce dependency on Spark-specific implementations
+ * </ul>
+ *
+ * @see ConversionTarget
+ * @see io.delta.kernel.Table
+ * @see io.delta.kernel.Transaction
+ */
+@Log4j2
+public class DeltaKernelConversionTarget implements ConversionTarget {
+  private static final String DELTA_LOG_RETENTION_DURATION = 
"delta.logRetentionDuration";
+
+  private DeltaKernelSchemaExtractor schemaExtractor;
+  private DeltaKernelPartitionExtractor partitionExtractor;
+  private DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor;
+
+  private String basePath;
+  private long logRetentionInHours;
+  private DeltaKernelConversionTarget.TransactionState transactionState;
+  private Engine engine;
+
+  /**
+   * No-arg constructor for ServiceLoader instantiation. Must call {@link 
#init(TargetTable,
+   * Configuration)} before use.
+   */
+  public DeltaKernelConversionTarget() {}
+
+  /**
+   * Creates a fully initialized DeltaKernelConversionTarget with custom 
Engine. Typically used in
+   * tests. Do not call {@link #init(TargetTable, Configuration)} after this.
+   *
+   * @param targetTable the target table configuration
+   * @param engine custom Delta Kernel engine instance
+   */
+  public DeltaKernelConversionTarget(TargetTable targetTable, Engine engine) {
+    this(
+        targetTable.getBasePath(),
+        targetTable.getMetadataRetention().toHours(),
+        engine,
+        DeltaKernelSchemaExtractor.getInstance(),
+        DeltaKernelPartitionExtractor.getInstance(),
+        DeltaKernelDataFileUpdatesExtractor.builder()
+            .engine(engine)
+            .basePath(targetTable.getBasePath())
+            // Column statistics are not needed for conversion operations
+            .includeColumnStats(false)
+            .build());
+  }
+
+  @VisibleForTesting
+  DeltaKernelConversionTarget(
+      String tableDataPath,
+      long logRetentionInHours,
+      Engine engine,
+      DeltaKernelSchemaExtractor schemaExtractor,
+      DeltaKernelPartitionExtractor partitionExtractor,
+      DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
+    _init(
+        tableDataPath,
+        logRetentionInHours,
+        engine,
+        schemaExtractor,
+        partitionExtractor,
+        dataKernelFileUpdatesExtractor);
+  }
+
+  /**
+   * Private initialization helper to avoid code duplication between 
constructor and init() paths.
+   */
+  private void _init(
+      String tableDataPath,
+      long logRetentionInHours,
+      Engine engine,
+      DeltaKernelSchemaExtractor schemaExtractor,
+      DeltaKernelPartitionExtractor partitionExtractor,
+      DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
+    this.basePath = tableDataPath;
+    this.schemaExtractor = schemaExtractor;
+    this.partitionExtractor = partitionExtractor;
+    this.dataKernelFileUpdatesExtractor = dataKernelFileUpdatesExtractor;
+    this.engine = engine;
+    this.logRetentionInHours = logRetentionInHours;
+  }
+
+  @Override
+  public void init(TargetTable targetTable, Configuration configuration) {
+    Engine engine = DefaultEngine.create(configuration);
+
+    _init(
+        targetTable.getBasePath(),
+        targetTable.getMetadataRetention().toHours(),
+        engine,
+        DeltaKernelSchemaExtractor.getInstance(),
+        DeltaKernelPartitionExtractor.getInstance(),
+        DeltaKernelDataFileUpdatesExtractor.builder()
+            .engine(engine)
+            .basePath(targetTable.getBasePath())
+            .includeColumnStats(true)
+            .build());
+  }
+
+  @Override
+  public void beginSync(InternalTable table) {
+    this.transactionState =
+        new DeltaKernelConversionTarget.TransactionState(engine, 
logRetentionInHours);
+  }
+
+  @Override
+  public void syncSchema(InternalSchema schema) {
+    transactionState.setLatestSchema(schema);
+  }
+
+  @Override
+  public void syncPartitionSpec(List<InternalPartitionField> partitionSpec) {
+    if (partitionSpec != null) {
+      Map<String, StructField> spec =
+          partitionExtractor.convertToDeltaPartitionFormat(partitionSpec);
+      for (Map.Entry<String, StructField> partitionEntry : spec.entrySet()) {
+        String partitionColumnName = partitionEntry.getKey();
+        StructField partitionField = partitionEntry.getValue();
+
+        transactionState.addPartitionColumn(partitionColumnName);
+        if (partitionField != null
+            && transactionState.getLatestSchema().fields().stream()
+                .noneMatch(field -> 
field.getName().equals(partitionField.getName()))) {
+          // add generated columns to schema.
+          transactionState.addColumn(partitionField);
+        }
+      }
+    }
+  }
+
+  @Override
+  public void syncMetadata(TableSyncMetadata metadata) {
+    transactionState.setMetadata(metadata);
+  }
+
+  @Override
+  public void syncFilesForSnapshot(List<PartitionFileGroup> 
partitionedDataFiles) {
+    Table table = Table.forPath(engine, basePath);
+    transactionState.setActions(
+        dataKernelFileUpdatesExtractor.applySnapshot(
+            table, partitionedDataFiles, 
transactionState.getLatestSchemaInternal()));
+  }
+
+  @Override
+  public void syncFilesForDiff(InternalFilesDiff internalFilesDiff) {
+    Table table = Table.forPath(engine, basePath);
+    transactionState.setActions(
+        dataKernelFileUpdatesExtractor.applyDiff(
+            internalFilesDiff, basePath, 
table.getLatestSnapshot(engine).getSchema()));
+  }
+
+  @Override
+  public void completeSync() {
+    transactionState.commitTransaction();
+    transactionState = null;
+  }
+
+  @Override
+  public Optional<TableSyncMetadata> getTableMetadata() {
+    Table table = Table.forPath(engine, basePath);
+    Snapshot snapshot = table.getLatestSnapshot(engine);
+
+    // WORKAROUND: Cast to SnapshotImpl (internal class) to access metadata 
configuration.
+    // Delta Kernel 4.0.0 does not provide a public API to access table 
metadata/configuration.
+    // This cast is brittle and may break on Kernel version upgrades.
+    Metadata metadata = ((SnapshotImpl) snapshot).getMetadata();
+
+    // Get configuration from metadata
+    Map<String, String> configuration = metadata.getConfiguration();
+    String metadataJson = configuration.get(TableSyncMetadata.XTABLE_METADATA);
+
+    return TableSyncMetadata.fromJson(metadataJson);
+  }
+
+  @Override
+  public String getTableFormat() {
+    return TableFormat.DELTA;
+  }
+
+  @Override
+  public Optional<String> getTargetCommitIdentifier(String sourceIdentifier) {
+    // Delta Kernel 4.0.0 does not support commit tags in commitInfo, which 
are required for
+    // source-to-target commit identifier mapping. This limitation is 
documented in:
+    // https://github.com/delta-io/delta/issues/6167
+    // XTable tracking issue: 
https://github.com/apache/incubator-xtable/issues/819
+    //
+    // Unlike DeltaConversionTarget (which uses Delta Standalone with commit 
tag support),
+    // DeltaKernelConversionTarget cannot retrieve commit tags from Delta 
Kernel's API.
+    // Rather than silently scanning all commits (O(n) performance cost) and 
always returning
+    // empty, we explicitly throw an exception to indicate this feature is 
unsupported.
+    //
+    // When Delta Kernel adds commit tag support, this method can be 
reimplemented to:
+    // 1. Scan commit history using tableImpl.getChanges(engine, 0, 
currentVersion, actionSet)
+    // 2. Extract tags from CommitInfo.tags MapValue
+    // 3. Parse XTABLE_METADATA from tags and match sourceIdentifier
+    throw new NotSupportedException(
+        "Source-to-target commit identifier mapping is not supported in 
DeltaKernelConversionTarget. "
+            + "Delta Kernel 4.0.0 does not support commit tags in commitInfo. "
+            + "See: https://github.com/delta-io/delta/issues/6167";);
+  }
+
+  private class TransactionState {
+    private final Engine engine;
+    private final long retentionInHours;
+    private final List<String> partitionColumns;
+    @Getter private StructType latestSchema;
+    @Getter private InternalSchema latestSchemaInternal;
+    private TableSyncMetadata metadata;
+    private List<RowBackedAction> actions;
+
+    private TransactionState(Engine engine, long retentionInHours) {
+      this.engine = engine;
+      this.partitionColumns = new ArrayList<>();
+      this.retentionInHours = retentionInHours;
+
+      try {
+        Table table = Table.forPath(engine, basePath);
+        this.latestSchema = table.getLatestSnapshot(engine).getSchema();

Review Comment:
   Multiple full snapshot loads per sync cycle: the constructor loads the 
snapshot here for schema, `commitTransaction()` calls `checkTableExists()` 
which loads another snapshot (line 470 -> `DeltaKernelUtils.tableExists`), and 
`syncFilesForSnapshot`/`syncFilesForDiff` load yet another (lines 268, 280). 
That's 3-4 full log replays per sync.
   
   Consider caching the snapshot (or at least the `tableExists` result) within 
a `TransactionState` lifecycle.



##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileUpdatesExtractor.java:
##########
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+package org.apache.xtable.kernel;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import lombok.Builder;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.MapValue;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.util.VectorUtils;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.exception.ReadException;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FilesDiff;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalFile;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.paths.PathUtils;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+@Builder
+public class DeltaKernelDataFileUpdatesExtractor {
+  @Builder.Default
+  private final DeltaKernelStatsExtractor deltaStatsExtractor =
+      DeltaKernelStatsExtractor.getInstance();
+
+  @Builder.Default
+  private final DeltaKernelPartitionExtractor deltaKernelPartitionExtractor =
+      DeltaKernelPartitionExtractor.getInstance();
+
+  @Builder.Default
+  private final DeltaKernelDataFileExtractor dataFileExtractor =
+      DeltaKernelDataFileExtractor.builder().build();
+
+  private final Engine engine;
+  private final String basePath;
+  private final boolean includeColumnStats;
+
+  public Seq<RowBackedAction> applySnapshot(
+      Table table, List<PartitionFileGroup> partitionedDataFiles, 
InternalSchema tableSchema) {
+
+    // all files in the current delta snapshot are potential candidates for 
remove actions, i.e. if
+    // the file is not present in the new snapshot (addedFiles) then the file 
is considered removed
+    Map<String, RowBackedAction> previousFiles = new HashMap<>();
+    StructType physicalSchema;
+
+    // Check if table exists by checking if _delta_log directory exists
+    boolean tableExists = checkTableExists(table);
+
+    if (tableExists) {
+      Snapshot snapshot = table.getLatestSnapshot(engine);
+
+      // Reuse DeltaKernelDataFileExtractor to iterate through existing files
+      // This avoids duplicating the scan logic for reading Delta files
+      try (DataFileIterator fileIterator =
+          dataFileExtractor.iterator(snapshot, table, engine, tableSchema)) {
+
+        while (fileIterator.hasNext()) {
+          InternalDataFile internalFile = fileIterator.next();
+
+          // Convert InternalDataFile back to AddFile to create RemoveFile 
action
+          AddFile addFile = createAddFileAction(internalFile, basePath, 
snapshot.getSchema());
+          RemoveFile removeFile =
+              new RemoveFile(addFile.toRemoveFileRow(false, 
Optional.of(snapshot.getVersion())));
+          String fullPath =
+              
DeltaKernelActionsConverter.getFullPathToFile(removeFile.getPath(), table);
+          previousFiles.put(fullPath, removeFile);
+        }
+      } catch (Exception e) {
+        throw new ReadException("Failed to scan existing Delta files", e);
+      }
+
+      physicalSchema = snapshot.getSchema();
+
+    } else {
+
+      // Table doesn't exist yet - no previous files to remove
+      // Convert InternalSchema to StructType for physical schema
+      DeltaKernelSchemaExtractor schemaExtractor = 
DeltaKernelSchemaExtractor.getInstance();
+      physicalSchema = schemaExtractor.fromInternalSchema(tableSchema);
+    }
+
+    FilesDiff<InternalFile, RowBackedAction> diff =
+        InternalFilesDiff.findNewAndRemovedFiles(partitionedDataFiles, 
previousFiles);
+
+    return applyDiff(diff.getFilesAdded(), diff.getFilesRemoved(), basePath, 
physicalSchema);
+  }
+
+  private boolean checkTableExists(Table table) {
+    return DeltaKernelUtils.tableExists(engine, table.getPath(engine));
+  }
+
+  public Seq<RowBackedAction> applyDiff(
+      InternalFilesDiff internalFilesDiff, String tableBasePath, StructType 
physicalSchema) {
+    List<RowBackedAction> removeActions =
+        internalFilesDiff.dataFilesRemoved().stream()
+            .map(dFile -> createAddFileAction(dFile, tableBasePath, 
physicalSchema))
+            .map(addFile -> new RemoveFile(addFile.toRemoveFileRow(false, 
Optional.empty())))
+            
.collect(CustomCollectors.toList(internalFilesDiff.dataFilesRemoved().size()));
+    return applyDiff(
+        internalFilesDiff.dataFilesAdded(), removeActions, tableBasePath, 
physicalSchema);
+  }
+
+  private Seq<RowBackedAction> applyDiff(
+      Set<? extends InternalFile> filesAdded,
+      Collection<RowBackedAction> removeFileActions,
+      String tableBasePath,
+      StructType physicalSchema) {
+    Stream<RowBackedAction> addActions =
+        filesAdded.stream()
+            .filter(InternalDataFile.class::isInstance)
+            .map(file -> (InternalDataFile) file)
+            .map(dFile -> createAddFileAction(dFile, tableBasePath, 
physicalSchema));
+    int totalActions = filesAdded.size() + removeFileActions.size();
+    List<RowBackedAction> allActions =
+        Stream.concat(addActions, removeFileActions.stream())
+            .collect(CustomCollectors.toList(totalActions));
+    return JavaConverters.asScalaBuffer(allActions).toSeq();
+  }
+
+  private AddFile createAddFileAction(
+      InternalDataFile dataFile, String tableBasePath, StructType 
physicalSchema) {
+    // Convert partition values from Map<String, String> to MapValue
+    Map<String, String> partitionValuesMap =
+        deltaKernelPartitionExtractor.partitionValueSerialization(dataFile);
+    MapValue partitionValues = convertToMapValue(partitionValuesMap);
+
+    Row addFileRow =
+        AddFile.createAddFileRow(
+            physicalSchema,
+            // Delta Lake supports relative and absolute paths in theory but 
relative paths seem
+            // more commonly supported by query engines in our testing
+            PathUtils.getRelativePath(dataFile.getPhysicalPath(), 
tableBasePath),
+            partitionValues,
+            dataFile.getFileSizeBytes(),
+            dataFile.getLastModified(),
+            true, // dataChange
+            Optional.empty(), // deletionVector
+            Optional.empty(), // tags
+            Optional.empty(), // baseRowId
+            Optional.empty(), // defaultRowCommitVersion
+            Optional.empty() // stats - TODO: convert column stats to 
DataFileStatistics

Review Comment:
   Stats are always `Optional.empty()`. The synced Delta table will have **no 
column statistics** on `AddFile` entries, degrading data skipping for 
downstream readers (Spark, Trino). Is there a tracking issue for adding stats 
support?



##########
xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelSync.java:
##########
@@ -0,0 +1,620 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+package org.apache.xtable.kernel;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.ScanImpl;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.conversion.TargetTable;
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.model.InternalSnapshot;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.metadata.TableSyncMetadata;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.model.schema.PartitionTransformType;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.model.storage.TableFormat;
+import org.apache.xtable.spi.sync.TableFormatSync;
+
+/**
+ * Validates that Delta Kernel tables are properly created/updated using
+ * DeltaKernelConversionTarget. Tests partitioning, schema evolution, and 
metadata sync without
+ * Spark SQL dependencies.
+ */
+public class TestDeltaKernelSync {
+  private static final Instant LAST_COMMIT_TIME = Instant.ofEpochSecond(1000);
+
+  @TempDir public Path tempDir;
+  private DeltaKernelConversionTarget conversionTarget;
+  private Path basePath;
+  private String tableName;
+  private Engine engine;
+
+  @BeforeEach
+  public void setup() throws IOException {
+    tableName = "test-" + UUID.randomUUID();
+    basePath = tempDir.resolve(tableName);
+    Files.createDirectories(basePath);
+
+    Configuration hadoopConf = new Configuration();
+    engine = DefaultEngine.create(hadoopConf);
+
+    conversionTarget =
+        new DeltaKernelConversionTarget(
+            TargetTable.builder()
+                .name(tableName)
+                .basePath(basePath.toString())
+                .metadataRetention(Duration.of(1, ChronoUnit.HOURS))
+                .formatName(TableFormat.DELTA)
+                .build(),
+            engine);
+  }
+
+  @Test
+  public void testCreateSnapshotControlFlow() throws Exception {
+    InternalSchema schema1 = getInternalSchema();
+    List<InternalField> fields2 = new ArrayList<>(schema1.getFields());
+    fields2.add(
+        InternalField.builder()
+            .name("float_field")
+            .schema(
+                InternalSchema.builder()
+                    .name("float")
+                    .dataType(InternalType.FLOAT)
+                    .isNullable(true)
+                    .build())
+            .build());
+    InternalSchema schema2 = 
getInternalSchema().toBuilder().fields(fields2).build();
+    InternalTable table1 = getInternalTable(tableName, basePath, schema1, 
null, LAST_COMMIT_TIME);
+    InternalTable table2 = getInternalTable(tableName, basePath, schema2, 
null, LAST_COMMIT_TIME);
+
+    InternalDataFile dataFile1 = getDataFile(1, Collections.emptyList(), 
basePath);
+    InternalDataFile dataFile2 = getDataFile(2, Collections.emptyList(), 
basePath);
+    InternalDataFile dataFile3 = getDataFile(3, Collections.emptyList(), 
basePath);
+
+    InternalSnapshot snapshot1 = buildSnapshot(table1, "0", dataFile1, 
dataFile2);
+    InternalSnapshot snapshot2 = buildSnapshot(table2, "1", dataFile2, 
dataFile3);
+
+    TableFormatSync.getInstance()
+        .syncSnapshot(Collections.singletonList(conversionTarget), snapshot1);
+    validateDeltaTable(basePath, new HashSet<>(Arrays.asList(dataFile1, 
dataFile2)));
+
+    TableFormatSync.getInstance()
+        .syncSnapshot(Collections.singletonList(conversionTarget), snapshot2);
+    validateDeltaTable(basePath, new HashSet<>(Arrays.asList(dataFile2, 
dataFile3)));
+  }
+
+  @Test
+  public void testFileRemovalWithCheckpoint() throws Exception {
+    // This test does 11 syncs to trigger checkpoint creation (happens at 10th 
commit)
+    // and verifies that file removal works correctly after checkpoint exists
+    String checkpointTableName = "test_table_checkpoint_" + UUID.randomUUID();
+    Path checkpointTestPath = tempDir.resolve(checkpointTableName);
+    Files.createDirectories(checkpointTestPath);
+
+    InternalSchema schema = getInternalSchema();
+    InternalTable checkpointTable =
+        getInternalTable(checkpointTableName, checkpointTestPath, schema, 
null, LAST_COMMIT_TIME);
+
+    DeltaKernelConversionTarget checkpointTarget =
+        new DeltaKernelConversionTarget(
+            TargetTable.builder()
+                .name(checkpointTableName)
+                .basePath(checkpointTestPath.toString())
+                .metadataRetention(Duration.of(1, ChronoUnit.HOURS))
+                .formatName(TableFormat.DELTA)
+                .build(),
+            engine);
+
+    // Do 10 syncs to trigger checkpoint
+    for (int i = 0; i < 10; i++) {
+      InternalDataFile file1 = getDataFile(i * 2 + 1, Collections.emptyList(), 
checkpointTestPath);
+      InternalDataFile file2 = getDataFile(i * 2 + 2, Collections.emptyList(), 
checkpointTestPath);
+
+      InternalSnapshot snapshot = buildSnapshot(checkpointTable, 
String.valueOf(i), file1, file2);
+      TableFormatSync.getInstance()
+          .syncSnapshot(Collections.singletonList(checkpointTarget), snapshot);
+    }
+
+    // 11th sync: This triggers checkpoint creation at version 10
+    InternalDataFile file21 = getDataFile(21, Collections.emptyList(), 
checkpointTestPath);
+    InternalDataFile file22 = getDataFile(22, Collections.emptyList(), 
checkpointTestPath);
+    InternalSnapshot snapshot11 = buildSnapshot(checkpointTable, "10", file21, 
file22);
+
+    TableFormatSync.getInstance()
+        .syncSnapshot(Collections.singletonList(checkpointTarget), snapshot11);
+
+    // Checkpoint is created synchronously via post-commit hooks
+    Path checkpointFile =
+        
checkpointTestPath.resolve("_delta_log/00000000000000000010.checkpoint.parquet");
+    assertTrue(Files.exists(checkpointFile), "Checkpoint file should exist 
after 10 commits");
+
+    // 12th sync: NOW checkpoint exists and can be used to detect file removals
+    InternalDataFile file23 = getDataFile(23, Collections.emptyList(), 
checkpointTestPath);
+    InternalDataFile file24 = getDataFile(24, Collections.emptyList(), 
checkpointTestPath);
+    InternalSnapshot snapshot12 = buildSnapshot(checkpointTable, "11", file23, 
file24);
+
+    TableFormatSync.getInstance()
+        .syncSnapshot(Collections.singletonList(checkpointTarget), snapshot12);
+
+    // Validate: Should only have file23 and file24 (file21/file22 should be 
removed)
+    validateDeltaTable(checkpointTestPath, new HashSet<>(Arrays.asList(file23, 
file24)));
+  }
+
+  @Test
+  public void testPrimitiveFieldPartitioning() throws Exception {
+    InternalSchema schema = getInternalSchema();
+    InternalPartitionField internalPartitionField =
+        InternalPartitionField.builder()
+            .sourceField(
+                InternalField.builder()
+                    .name("string_field")
+                    .schema(
+                        InternalSchema.builder()
+                            .name("string")
+                            .dataType(InternalType.STRING)
+                            .build())
+                    .build())
+            .transformType(PartitionTransformType.VALUE)
+            .build();
+    InternalTable table =
+        getInternalTable(
+            tableName,
+            basePath,
+            schema,
+            Collections.singletonList(internalPartitionField),
+            LAST_COMMIT_TIME);
+
+    List<PartitionValue> partitionValues1 =
+        Collections.singletonList(
+            PartitionValue.builder()
+                .partitionField(internalPartitionField)
+                .range(Range.scalar("level"))
+                .build());
+    List<PartitionValue> partitionValues2 =
+        Collections.singletonList(
+            PartitionValue.builder()
+                .partitionField(internalPartitionField)
+                .range(Range.scalar("warning"))
+                .build());
+    InternalDataFile dataFile1 = getDataFile(1, partitionValues1, basePath);
+    InternalDataFile dataFile2 = getDataFile(2, partitionValues1, basePath);
+    InternalDataFile dataFile3 = getDataFile(3, partitionValues2, basePath);
+
+    InternalSnapshot snapshot1 = buildSnapshot(table, "0", dataFile1, 
dataFile2, dataFile3);
+
+    TableFormatSync.getInstance()
+        .syncSnapshot(Collections.singletonList(conversionTarget), snapshot1);
+
+    // Validate all files are present
+    validateDeltaTable(basePath, new HashSet<>(Arrays.asList(dataFile1, 
dataFile2, dataFile3)));
+
+    // Verify partition columns are set
+    Table deltaTable = Table.forPath(engine, basePath.toString());
+    Snapshot snapshot = deltaTable.getLatestSnapshot(engine);
+    SnapshotImpl snapshotImpl = (SnapshotImpl) snapshot;
+    Set<String> partitionColumns = 
snapshotImpl.getMetadata().getPartitionColNames();
+    assertEquals(1, partitionColumns.size());
+    assertTrue(partitionColumns.contains("string_field"));
+  }
+
+  @Test
+  public void testMultipleFieldPartitioning() throws Exception {
+    InternalSchema schema = getInternalSchema();
+    InternalPartitionField internalPartitionField1 =
+        InternalPartitionField.builder()
+            .sourceField(
+                InternalField.builder()
+                    .name("string_field")
+                    .schema(
+                        InternalSchema.builder()
+                            .name("string")
+                            .dataType(InternalType.STRING)
+                            .build())
+                    .build())
+            .transformType(PartitionTransformType.VALUE)
+            .build();
+    InternalPartitionField internalPartitionField2 =
+        InternalPartitionField.builder()
+            .sourceField(
+                InternalField.builder()
+                    .name("int_field")
+                    
.schema(InternalSchema.builder().name("int").dataType(InternalType.INT).build())
+                    .build())
+            .transformType(PartitionTransformType.VALUE)
+            .build();
+    InternalTable table =
+        getInternalTable(
+            tableName,
+            basePath,
+            schema,
+            Arrays.asList(internalPartitionField1, internalPartitionField2),
+            LAST_COMMIT_TIME);
+
+    List<PartitionValue> partitionValues1 =
+        Arrays.asList(
+            PartitionValue.builder()
+                .partitionField(internalPartitionField1)
+                .range(Range.scalar("level"))
+                .build(),
+            PartitionValue.builder()
+                .partitionField(internalPartitionField2)
+                .range(Range.scalar(10))
+                .build());
+    List<PartitionValue> partitionValues2 =
+        Arrays.asList(
+            PartitionValue.builder()
+                .partitionField(internalPartitionField1)
+                .range(Range.scalar("level"))
+                .build(),
+            PartitionValue.builder()
+                .partitionField(internalPartitionField2)
+                .range(Range.scalar(20))
+                .build());
+    List<PartitionValue> partitionValues3 =
+        Arrays.asList(
+            PartitionValue.builder()
+                .partitionField(internalPartitionField1)
+                .range(Range.scalar("warning"))
+                .build(),
+            PartitionValue.builder()
+                .partitionField(internalPartitionField2)
+                .range(Range.scalar(20))
+                .build());
+
+    InternalDataFile dataFile1 = getDataFile(1, partitionValues1, basePath);
+    InternalDataFile dataFile2 = getDataFile(2, partitionValues2, basePath);
+    InternalDataFile dataFile3 = getDataFile(3, partitionValues3, basePath);
+
+    InternalSnapshot snapshot1 = buildSnapshot(table, "0", dataFile1, 
dataFile2, dataFile3);
+    TableFormatSync.getInstance()
+        .syncSnapshot(Collections.singletonList(conversionTarget), snapshot1);
+    validateDeltaTable(basePath, new HashSet<>(Arrays.asList(dataFile1, 
dataFile2, dataFile3)));
+
+    // Verify partition columns
+    Table deltaTable = Table.forPath(engine, basePath.toString());
+    Snapshot snapshot = deltaTable.getLatestSnapshot(engine);
+    SnapshotImpl snapshotImpl = (SnapshotImpl) snapshot;
+    Set<String> partitionColumns = 
snapshotImpl.getMetadata().getPartitionColNames();
+    assertEquals(2, partitionColumns.size());
+    assertTrue(partitionColumns.contains("string_field"));
+    assertTrue(partitionColumns.contains("int_field"));
+  }
+
+  @Test
+  @Disabled(
+      "Disabled due to tags not present in commitinfo - 
https://github.com/delta-io/delta/issues/6167";)
+  public void testSourceTargetIdMapping() throws Exception {
+    InternalSchema baseSchema = getInternalSchema();
+    InternalTable sourceTable =
+        getInternalTable("source_table", basePath, baseSchema, null, 
LAST_COMMIT_TIME);
+
+    InternalDataFile sourceDataFile1 = getDataFile(101, 
Collections.emptyList(), basePath);
+    InternalDataFile sourceDataFile2 = getDataFile(102, 
Collections.emptyList(), basePath);
+    InternalDataFile sourceDataFile3 = getDataFile(103, 
Collections.emptyList(), basePath);
+
+    InternalSnapshot sourceSnapshot1 =
+        buildSnapshot(sourceTable, "0", sourceDataFile1, sourceDataFile2);
+    InternalSnapshot sourceSnapshot2 =
+        buildSnapshot(sourceTable, "1", sourceDataFile2, sourceDataFile3);
+
+    TableFormatSync.getInstance()
+        .syncSnapshot(Collections.singletonList(conversionTarget), 
sourceSnapshot1);
+    Optional<String> mappedTargetId1 =
+        
conversionTarget.getTargetCommitIdentifier(sourceSnapshot1.getSourceIdentifier());
+    validateDeltaTable(basePath, new HashSet<>(Arrays.asList(sourceDataFile1, 
sourceDataFile2)));
+    assertTrue(mappedTargetId1.isPresent());
+    assertEquals("0", mappedTargetId1.get());
+
+    TableFormatSync.getInstance()
+        .syncSnapshot(Collections.singletonList(conversionTarget), 
sourceSnapshot2);
+    Optional<String> mappedTargetId2 =
+        
conversionTarget.getTargetCommitIdentifier(sourceSnapshot2.getSourceIdentifier());
+    validateDeltaTable(basePath, new HashSet<>(Arrays.asList(sourceDataFile2, 
sourceDataFile3)));
+    assertTrue(mappedTargetId2.isPresent());
+    assertEquals("1", mappedTargetId2.get());
+
+    Optional<String> unmappedTargetId = 
conversionTarget.getTargetCommitIdentifier("s3");
+    assertFalse(unmappedTargetId.isPresent());
+  }
+
+  @Test
+  public void testGetTargetCommitIdentifierWithNullSourceIdentifier() throws 
Exception {
+    InternalSchema baseSchema = getInternalSchema();
+    InternalTable internalTable =
+        getInternalTable("source_table", basePath, baseSchema, null, 
LAST_COMMIT_TIME);
+    InternalDataFile sourceDataFile = getDataFile(101, 
Collections.emptyList(), basePath);
+    InternalSnapshot snapshot = buildSnapshot(internalTable, "0", 
sourceDataFile);
+
+    // Mock the snapshot sync process
+    conversionTarget.beginSync(internalTable);
+    TableSyncMetadata tableSyncMetadata =
+        TableSyncMetadata.of(
+            internalTable.getLatestCommitTime(), new 
ArrayList<>(snapshot.getPendingCommits()));
+    conversionTarget.syncMetadata(tableSyncMetadata);
+    conversionTarget.syncSchema(internalTable.getReadSchema());
+    conversionTarget.syncPartitionSpec(internalTable.getPartitioningFields());
+    conversionTarget.syncFilesForSnapshot(snapshot.getPartitionedDataFiles());
+    conversionTarget.completeSync();
+
+    // getTargetCommitIdentifier is not supported in 
DeltaKernelConversionTarget
+    // because Delta Kernel 4.0.0 does not support commit tags
+    NotSupportedException exception =
+        assertThrows(
+            NotSupportedException.class, () -> 
conversionTarget.getTargetCommitIdentifier("0"));
+    assertTrue(
+        exception
+            .getMessage()
+            .contains("Source-to-target commit identifier mapping is not 
supported"));
+  }
+
+  @Test
+  public void testGetTableMetadata() throws Exception {
+    InternalSchema schema = getInternalSchema();
+    InternalTable table = getInternalTable(tableName, basePath, schema, null, 
LAST_COMMIT_TIME);
+    InternalDataFile dataFile = getDataFile(1, Collections.emptyList(), 
basePath);
+    InternalSnapshot snapshot = buildSnapshot(table, "0", dataFile);
+
+    TableFormatSync.getInstance()
+        .syncSnapshot(Collections.singletonList(conversionTarget), snapshot);
+
+    Optional<TableSyncMetadata> metadata = conversionTarget.getTableMetadata();
+    assertTrue(metadata.isPresent(), "Metadata should be present after sync");
+    TableSyncMetadata syncMetadata = metadata.get();
+    assertNotNull(syncMetadata.getLastInstantSynced(), "Last instant synced 
should not be null");
+  }
+
+  private void validateDeltaTable(Path basePath, Set<InternalDataFile> 
expectedFiles)
+      throws IOException {
+    Table table = Table.forPath(engine, basePath.toString());
+    assertNotNull(table);
+
+    Snapshot snapshot = table.getLatestSnapshot(engine);
+    assertNotNull(snapshot);
+
+    // Scan all files
+    ScanImpl scan = (ScanImpl) snapshot.getScanBuilder().build();
+    CloseableIterator<FilteredColumnarBatch> scanFiles = 
scan.getScanFiles(engine, false);

Review Comment:
   `validateDeltaTable` uses `CloseableIterator<FilteredColumnarBatch>` (line 
440) and `CloseableIterator<Row>` (line 449) that are never closed. Use 
try-with-resources to prevent resource leaks:
   ```java
   try (CloseableIterator<FilteredColumnarBatch> scanFiles = 
scan.getScanFiles(engine, false)) {
       // ...
   }
   ```



##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelConversionTarget.java:
##########
@@ -0,0 +1,483 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+package org.apache.xtable.kernel;
+
+import static io.delta.kernel.utils.CloseableIterable.inMemoryIterable;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import lombok.Getter;
+import lombok.extern.log4j.Log4j2;
+
+import org.apache.hadoop.conf.Configuration;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import io.delta.kernel.Operation;
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.Transaction;
+import io.delta.kernel.TransactionBuilder;
+import io.delta.kernel.TransactionCommitResult;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.exceptions.TableNotFoundException;
+import io.delta.kernel.hook.PostCommitHook;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.Metadata;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.actions.SingleAction;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterable;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.conversion.TargetTable;
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.exception.UpdateException;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.metadata.TableSyncMetadata;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.model.storage.TableFormat;
+import org.apache.xtable.spi.sync.ConversionTarget;
+
+/**
+ * Implementation of {@link ConversionTarget} for Delta Lake using the Delta 
Kernel API.
+ *
+ * <p>This implementation uses Delta Kernel (io.delta.kernel) instead of Delta 
Standalone for write
+ * operations, providing better compatibility with cloud storage (S3, GCS, 
Azure Blob Storage, HDFS)
+ * and improved support for Delta Lake 3.x features.
+ *
+ * <p><strong>Initialization:</strong> This class supports two initialization 
patterns:
+ *
+ * <ul>
+ *   <li><strong>Factory/ServiceLoader Pattern:</strong> Use the no-arg 
constructor followed by
+ *       {@link #init(TargetTable, Configuration)}. This is used by {@link
+ *       org.apache.xtable.conversion.ConversionTargetFactory}.
+ *   <li><strong>Direct Testing Pattern:</strong> Use the constructor with 
{@link TargetTable} and
+ *       {@link Engine} parameters for direct instantiation with custom 
dependencies in tests.
+ * </ul>
+ *
+ * <p><strong>Important:</strong> Do not mix initialization patterns. If you 
use the parameterized
+ * constructor, do not call {@link #init(TargetTable, Configuration)} 
afterward, as it will
+ * overwrite the custom Engine.
+ *
+ * <p><strong>Exception Handling:</strong> This implementation only catches 
{@link
+ * io.delta.kernel.exceptions.TableNotFoundException} when checking for table 
existence, allowing
+ * other exceptions (network errors, permission issues, corrupted metadata) to 
propagate rather than
+ * being silently masked. This ensures real errors are visible and fail fast.
+ *
+ * <p><strong>Known Limitations:</strong>
+ *
+ * <ul>
+ *   <li><strong>Commit Tags:</strong> Delta Kernel 4.0.0 does not support 
commit tags in commitInfo
+ *       (e.g., XTABLE_METADATA tags). This affects source-to-target commit 
identifier mapping.
+ *       Tracked in: https://github.com/apache/incubator-xtable/issues/819
+ *   <li><strong>Schema Evolution:</strong> Schema changes are handled through 
Delta Kernel's
+ *       transaction API, which may have different semantics compared to Delta 
Standalone.
+ *   <li><strong>Internal API Usage:</strong> This implementation casts to 
internal classes
+ *       (SnapshotImpl, TableImpl) to access metadata and commit history, as 
Delta Kernel 4.0.0
+ *       lacks public APIs for these operations. These casts are brittle and 
may break on version
+ *       upgrades. Public API alternatives should be used when available.
+ * </ul>
+ *
+ * <p><strong>Implementation Choice:</strong> Delta Kernel API was chosen over 
Delta Standalone to:
+ *
+ * <ul>
+ *   <li>Support newer Delta Lake features and protocol versions
+ *   <li>Align with the Delta Lake community's direction (Kernel is the 
recommended API)
+ *   <li>Reduce dependency on Spark-specific implementations
+ * </ul>
+ *
+ * @see ConversionTarget
+ * @see io.delta.kernel.Table
+ * @see io.delta.kernel.Transaction
+ */
+@Log4j2
+public class DeltaKernelConversionTarget implements ConversionTarget {
+  private static final String DELTA_LOG_RETENTION_DURATION = 
"delta.logRetentionDuration";
+
+  private DeltaKernelSchemaExtractor schemaExtractor;
+  private DeltaKernelPartitionExtractor partitionExtractor;
+  private DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor;
+
+  private String basePath;
+  private long logRetentionInHours;
+  private DeltaKernelConversionTarget.TransactionState transactionState;
+  private Engine engine;
+
+  /**
+   * No-arg constructor for ServiceLoader instantiation. Must call {@link 
#init(TargetTable,
+   * Configuration)} before use.
+   */
+  public DeltaKernelConversionTarget() {}
+
+  /**
+   * Creates a fully initialized DeltaKernelConversionTarget with custom 
Engine. Typically used in
+   * tests. Do not call {@link #init(TargetTable, Configuration)} after this.
+   *
+   * @param targetTable the target table configuration
+   * @param engine custom Delta Kernel engine instance
+   */
+  public DeltaKernelConversionTarget(TargetTable targetTable, Engine engine) {
+    this(
+        targetTable.getBasePath(),
+        targetTable.getMetadataRetention().toHours(),
+        engine,
+        DeltaKernelSchemaExtractor.getInstance(),
+        DeltaKernelPartitionExtractor.getInstance(),
+        DeltaKernelDataFileUpdatesExtractor.builder()
+            .engine(engine)
+            .basePath(targetTable.getBasePath())
+            // Column statistics are not needed for conversion operations
+            .includeColumnStats(false)
+            .build());
+  }
+
+  @VisibleForTesting
+  DeltaKernelConversionTarget(
+      String tableDataPath,
+      long logRetentionInHours,
+      Engine engine,
+      DeltaKernelSchemaExtractor schemaExtractor,
+      DeltaKernelPartitionExtractor partitionExtractor,
+      DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
+    _init(
+        tableDataPath,
+        logRetentionInHours,
+        engine,
+        schemaExtractor,
+        partitionExtractor,
+        dataKernelFileUpdatesExtractor);
+  }
+
+  /**
+   * Private initialization helper to avoid code duplication between 
constructor and init() paths.
+   */
+  private void _init(
+      String tableDataPath,
+      long logRetentionInHours,
+      Engine engine,
+      DeltaKernelSchemaExtractor schemaExtractor,
+      DeltaKernelPartitionExtractor partitionExtractor,
+      DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
+    this.basePath = tableDataPath;
+    this.schemaExtractor = schemaExtractor;
+    this.partitionExtractor = partitionExtractor;
+    this.dataKernelFileUpdatesExtractor = dataKernelFileUpdatesExtractor;
+    this.engine = engine;
+    this.logRetentionInHours = logRetentionInHours;
+  }
+
+  @Override
+  public void init(TargetTable targetTable, Configuration configuration) {
+    Engine engine = DefaultEngine.create(configuration);
+
+    _init(
+        targetTable.getBasePath(),
+        targetTable.getMetadataRetention().toHours(),
+        engine,
+        DeltaKernelSchemaExtractor.getInstance(),
+        DeltaKernelPartitionExtractor.getInstance(),
+        DeltaKernelDataFileUpdatesExtractor.builder()
+            .engine(engine)
+            .basePath(targetTable.getBasePath())
+            .includeColumnStats(true)
+            .build());
+  }
+
+  @Override
+  public void beginSync(InternalTable table) {
+    this.transactionState =
+        new DeltaKernelConversionTarget.TransactionState(engine, 
logRetentionInHours);
+  }
+
+  @Override
+  public void syncSchema(InternalSchema schema) {
+    transactionState.setLatestSchema(schema);
+  }
+
+  @Override
+  public void syncPartitionSpec(List<InternalPartitionField> partitionSpec) {
+    if (partitionSpec != null) {
+      Map<String, StructField> spec =
+          partitionExtractor.convertToDeltaPartitionFormat(partitionSpec);
+      for (Map.Entry<String, StructField> partitionEntry : spec.entrySet()) {
+        String partitionColumnName = partitionEntry.getKey();
+        StructField partitionField = partitionEntry.getValue();
+
+        transactionState.addPartitionColumn(partitionColumnName);
+        if (partitionField != null
+            && transactionState.getLatestSchema().fields().stream()
+                .noneMatch(field -> 
field.getName().equals(partitionField.getName()))) {
+          // add generated columns to schema.
+          transactionState.addColumn(partitionField);
+        }
+      }
+    }
+  }
+
+  @Override
+  public void syncMetadata(TableSyncMetadata metadata) {
+    transactionState.setMetadata(metadata);
+  }
+
+  @Override
+  public void syncFilesForSnapshot(List<PartitionFileGroup> 
partitionedDataFiles) {
+    Table table = Table.forPath(engine, basePath);
+    transactionState.setActions(
+        dataKernelFileUpdatesExtractor.applySnapshot(
+            table, partitionedDataFiles, 
transactionState.getLatestSchemaInternal()));
+  }
+
+  @Override
+  public void syncFilesForDiff(InternalFilesDiff internalFilesDiff) {
+    Table table = Table.forPath(engine, basePath);
+    transactionState.setActions(
+        dataKernelFileUpdatesExtractor.applyDiff(
+            internalFilesDiff, basePath, 
table.getLatestSnapshot(engine).getSchema()));
+  }
+
+  @Override
+  public void completeSync() {
+    transactionState.commitTransaction();
+    transactionState = null;
+  }
+
+  @Override
+  public Optional<TableSyncMetadata> getTableMetadata() {
+    Table table = Table.forPath(engine, basePath);
+    Snapshot snapshot = table.getLatestSnapshot(engine);
+
+    // WORKAROUND: Cast to SnapshotImpl (internal class) to access metadata 
configuration.
+    // Delta Kernel 4.0.0 does not provide a public API to access table 
metadata/configuration.
+    // This cast is brittle and may break on Kernel version upgrades.
+    Metadata metadata = ((SnapshotImpl) snapshot).getMetadata();
+
+    // Get configuration from metadata
+    Map<String, String> configuration = metadata.getConfiguration();
+    String metadataJson = configuration.get(TableSyncMetadata.XTABLE_METADATA);
+
+    return TableSyncMetadata.fromJson(metadataJson);
+  }
+
+  @Override
+  public String getTableFormat() {
+    return TableFormat.DELTA;
+  }
+
+  @Override
+  public Optional<String> getTargetCommitIdentifier(String sourceIdentifier) {
+    // Delta Kernel 4.0.0 does not support commit tags in commitInfo, which 
are required for
+    // source-to-target commit identifier mapping. This limitation is 
documented in:
+    // https://github.com/delta-io/delta/issues/6167
+    // XTable tracking issue: 
https://github.com/apache/incubator-xtable/issues/819
+    //
+    // Unlike DeltaConversionTarget (which uses Delta Standalone with commit 
tag support),
+    // DeltaKernelConversionTarget cannot retrieve commit tags from Delta 
Kernel's API.
+    // Rather than silently scanning all commits (O(n) performance cost) and 
always returning
+    // empty, we explicitly throw an exception to indicate this feature is 
unsupported.
+    //
+    // When Delta Kernel adds commit tag support, this method can be 
reimplemented to:
+    // 1. Scan commit history using tableImpl.getChanges(engine, 0, 
currentVersion, actionSet)
+    // 2. Extract tags from CommitInfo.tags MapValue
+    // 3. Parse XTABLE_METADATA from tags and match sourceIdentifier
+    throw new NotSupportedException(
+        "Source-to-target commit identifier mapping is not supported in 
DeltaKernelConversionTarget. "
+            + "Delta Kernel 4.0.0 does not support commit tags in commitInfo. "
+            + "See: https://github.com/delta-io/delta/issues/6167";);
+  }
+
+  private class TransactionState {
+    private final Engine engine;
+    private final long retentionInHours;
+    private final List<String> partitionColumns;
+    @Getter private StructType latestSchema;
+    @Getter private InternalSchema latestSchemaInternal;
+    private TableSyncMetadata metadata;
+    private List<RowBackedAction> actions;
+
+    private TransactionState(Engine engine, long retentionInHours) {
+      this.engine = engine;
+      this.partitionColumns = new ArrayList<>();
+      this.retentionInHours = retentionInHours;
+
+      try {
+        Table table = Table.forPath(engine, basePath);
+        this.latestSchema = table.getLatestSnapshot(engine).getSchema();
+      } catch (TableNotFoundException e) {
+        // Expected: table doesn't exist yet on first sync
+        this.latestSchema = null;
+      }
+      // Let other exceptions propagate (network issues, permissions, 
corrupted metadata, etc.)
+    }
+
+    /**
+     * Adds a partition column name to the list. Package-private to allow 
access from outer class.
+     */
+    void addPartitionColumn(String columnName) {
+      partitionColumns.add(columnName);
+    }
+
+    void setMetadata(TableSyncMetadata metadata) {
+      this.metadata = metadata;
+    }
+
+    /**
+     * Sets the actions to be committed. Converts from Scala Seq to Java List 
for internal storage.
+     */
+    void setActions(Seq<RowBackedAction> scalaActions) {
+      this.actions = JavaConverters.seqAsJavaList(scalaActions);
+    }
+
+    private void addColumn(StructField field) {
+      latestSchema = latestSchema.add(field);
+      latestSchemaInternal = schemaExtractor.toInternalSchema(latestSchema);
+    }
+
+    private void setLatestSchema(InternalSchema schema) {
+      this.latestSchemaInternal = schema;
+      this.latestSchema = schemaExtractor.fromInternalSchema(schema);
+    }
+
+    private void commitTransaction() {
+      boolean tableExists = checkTableExists();
+
+      Operation operation = tableExists ? Operation.WRITE : 
Operation.CREATE_TABLE;
+
+      if (!tableExists) {
+        File tableDir = new File(basePath);
+        if (!tableDir.exists()) {
+          tableDir.mkdirs();

Review Comment:
   `new File(basePath).mkdirs()` only works for local filesystem paths. For 
S3/GCS/Azure/HDFS paths (e.g., `s3://bucket/path`), this silently fails or 
creates nonsensical local directories.
   
   Consider removing this block — `Table.forPath()` with `CREATE_TABLE` 
operation should handle table directory creation. Or use Hadoop 
`FileSystem.mkdirs()` if explicit creation is needed.



##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileUpdatesExtractor.java:
##########
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+package org.apache.xtable.kernel;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import lombok.Builder;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.MapValue;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.util.VectorUtils;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.exception.ReadException;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FilesDiff;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalFile;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.paths.PathUtils;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+@Builder
+public class DeltaKernelDataFileUpdatesExtractor {
+  @Builder.Default
+  private final DeltaKernelStatsExtractor deltaStatsExtractor =
+      DeltaKernelStatsExtractor.getInstance();
+
+  @Builder.Default
+  private final DeltaKernelPartitionExtractor deltaKernelPartitionExtractor =
+      DeltaKernelPartitionExtractor.getInstance();
+
+  @Builder.Default
+  private final DeltaKernelDataFileExtractor dataFileExtractor =
+      DeltaKernelDataFileExtractor.builder().build();
+
+  private final Engine engine;
+  private final String basePath;
+  private final boolean includeColumnStats;
+
+  public Seq<RowBackedAction> applySnapshot(
+      Table table, List<PartitionFileGroup> partitionedDataFiles, 
InternalSchema tableSchema) {
+
+    // all files in the current delta snapshot are potential candidates for 
remove actions, i.e. if
+    // the file is not present in the new snapshot (addedFiles) then the file 
is considered removed
+    Map<String, RowBackedAction> previousFiles = new HashMap<>();
+    StructType physicalSchema;
+
+    // Check if table exists by checking if _delta_log directory exists
+    boolean tableExists = checkTableExists(table);
+
+    if (tableExists) {
+      Snapshot snapshot = table.getLatestSnapshot(engine);
+
+      // Reuse DeltaKernelDataFileExtractor to iterate through existing files
+      // This avoids duplicating the scan logic for reading Delta files
+      try (DataFileIterator fileIterator =
+          dataFileExtractor.iterator(snapshot, table, engine, tableSchema)) {
+
+        while (fileIterator.hasNext()) {
+          InternalDataFile internalFile = fileIterator.next();
+
+          // Convert InternalDataFile back to AddFile to create RemoveFile 
action
+          AddFile addFile = createAddFileAction(internalFile, basePath, 
snapshot.getSchema());
+          RemoveFile removeFile =
+              new RemoveFile(addFile.toRemoveFileRow(false, 
Optional.of(snapshot.getVersion())));
+          String fullPath =
+              
DeltaKernelActionsConverter.getFullPathToFile(removeFile.getPath(), table);

Review Comment:
   Performance concern: `DeltaKernelActionsConverter.getFullPathToFile()` 
creates a **new `Configuration` + `DefaultEngine` per call**. In 
`applySnapshot`, this is invoked per-file in the existing snapshot — 
potentially thousands of times. Each `new Configuration()` scans the classpath.
   
   Consider accepting an `Engine` parameter, or computing the full path 
directly from `basePath + removeFile.getPath()` without constructing a new 
engine each time.



##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileUpdatesExtractor.java:
##########
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+package org.apache.xtable.kernel;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import lombok.Builder;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.MapValue;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.util.VectorUtils;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.exception.ReadException;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FilesDiff;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalFile;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.paths.PathUtils;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+@Builder
+public class DeltaKernelDataFileUpdatesExtractor {
+  @Builder.Default
+  private final DeltaKernelStatsExtractor deltaStatsExtractor =
+      DeltaKernelStatsExtractor.getInstance();
+
+  @Builder.Default
+  private final DeltaKernelPartitionExtractor deltaKernelPartitionExtractor =
+      DeltaKernelPartitionExtractor.getInstance();
+
+  @Builder.Default
+  private final DeltaKernelDataFileExtractor dataFileExtractor =
+      DeltaKernelDataFileExtractor.builder().build();
+
+  private final Engine engine;
+  private final String basePath;
+  private final boolean includeColumnStats;

Review Comment:
   `includeColumnStats` field is declared but **never read** — 
`createAddFileAction` always passes `Optional.empty()` for stats (line 177). 
Either implement stats conversion or remove the dead field. If deferred, add a 
`// TODO(issue-link)` so it doesn't get forgotten.



##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelStatsExtractor.java:
##########
@@ -265,22 +265,27 @@ private void collectUnsupportedStats(Map<String, Object> 
additionalStats) {
    */

Review Comment:
   This class is ~300 lines of near-identical duplication from 
`DeltaStatsExtractor` in the Standalone package. The only material difference 
is the `AddFile` import. Consider extracting shared stats serialization logic 
(`convertStatsToDeltaFormat`, `insertValueAtPath`, `flattenStatMap`, 
`DeltaStats`) into a common utility to avoid maintaining two copies.



##########
xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelReadWriteIntegration.java:
##########
@@ -0,0 +1,488 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ 
+package org.apache.xtable.kernel;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.SnapshotImpl;
+
+import org.apache.xtable.conversion.TargetTable;
+import org.apache.xtable.model.InternalSnapshot;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.model.schema.PartitionTransformType;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.model.storage.TableFormat;
+import org.apache.xtable.spi.sync.TableFormatSync;
+
+/**
+ * Comprehensive end-to-end integration test for Delta Kernel read and write 
operations.
+ *
+ * <p>This test validates: 1. Writing data to Delta tables using 
DeltaKernelConversionTarget 2.
+ * Reading data from Delta tables using DeltaKernelConversionSource 3. 
Round-trip data integrity
+ * (write → read → validate) 4. Partitioned tables 5. Incremental updates 
(add/remove files) 6. Time
+ * travel (version-based reads) 7. Empty table handling
+ */
+public class TestDeltaKernelReadWriteIntegration {
+  private static final Random RANDOM = new Random();
+  private static final Instant LAST_COMMIT_TIME = Instant.now();
+
+  @TempDir public Path tempDir;
+  private Engine engine;
+
+  @BeforeEach
+  public void setup() {
+    Configuration hadoopConf = new Configuration();
+    engine = DefaultEngine.create(hadoopConf);
+  }
+
+  /**
+   * Test 1: Basic Write and Read Validates that data written to Delta can be 
read back correctly.
+   */
+  @Test
+  public void testBasicWriteAndRead() throws Exception {
+    String tableName = "test_basic_" + UUID.randomUUID();
+    Path basePath = tempDir.resolve(tableName);
+    Files.createDirectories(basePath);
+
+    // === WRITE PHASE ===
+    InternalSchema schema = createSimpleSchema();
+    DeltaKernelConversionTarget writer = createWriter(tableName, basePath);
+
+    // Create test data files
+    InternalDataFile file1 = createDataFile(1, Collections.emptyList(), 
basePath);
+    InternalDataFile file2 = createDataFile(2, Collections.emptyList(), 
basePath);
+
+    // Write data to Delta table
+    InternalTable writeTable = createInternalTable(tableName, basePath, 
schema, null);
+    InternalSnapshot snapshot = buildSnapshot(writeTable, "0", file1, file2);
+    
TableFormatSync.getInstance().syncSnapshot(Collections.singletonList(writer), 
snapshot);
+
+    // Verify Delta table was created
+    assertTrue(Files.exists(basePath.resolve("_delta_log")), "Delta log 
directory should exist");
+
+    // === READ PHASE ===
+    DeltaKernelConversionSource reader = createReader(tableName, basePath);
+
+    // Read current table metadata
+    InternalTable readTable = reader.getCurrentTable();
+    assertNotNull(readTable, "Should be able to read table");
+    assertEquals(tableName, readTable.getName());
+
+    // Normalize paths for comparison (handle file:// prefix differences)
+    String expectedPath = basePath.toString();
+    String actualPath = readTable.getBasePath().replace("file://", 
"").replace("file:", "");
+    assertTrue(
+        actualPath.endsWith(expectedPath) || actualPath.equals(expectedPath),
+        "Base path should match. Expected: " + expectedPath + ", Actual: " + 
actualPath);
+
+    // Verify schema
+    InternalSchema readSchema = readTable.getReadSchema();
+    assertNotNull(readSchema);
+    assertEquals(schema.getFields().size(), readSchema.getFields().size());

Review Comment:
   nit: This only checks field count, not content. Use `assertEquals(schema, 
readSchema)` for full structural comparison — otherwise schema corruption that 
preserves field count goes undetected.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to