the-other-tim-brown commented on code in PR #801:
URL: https://github.com/apache/incubator-xtable/pull/801#discussion_r3006327885
##########
xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelSchemaExtractor.java:
##########
@@ -873,4 +883,142 @@ public void testIcebergToDeltaUUIDSupport() {
internalSchema,
DeltaKernelSchemaExtractor.getInstance().toInternalSchema(structRepresentation));
}
+
+ // ========== Tests for fromInternalSchema() - New Tests ==========
Review Comment:
Remove this comment for `New Tests`?
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileUpdatesExtractor.java:
##########
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import lombok.Builder;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.MapValue;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.util.VectorUtils;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.exception.ReadException;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FilesDiff;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalFile;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.paths.PathUtils;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+@Builder
+public class DeltaKernelDataFileUpdatesExtractor {
+ @Builder.Default
+ private final DeltaKernelStatsExtractor deltaStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor deltaKernelPartitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelDataFileExtractor dataFileExtractor =
+ DeltaKernelDataFileExtractor.builder().build();
+
+ private final Engine engine;
+ private final String basePath;
+ private final boolean includeColumnStats;
+
+ public Seq<RowBackedAction> applySnapshot(
+ Table table, List<PartitionFileGroup> partitionedDataFiles,
InternalSchema tableSchema) {
+
+ // all files in the current delta snapshot are potential candidates for
remove actions, i.e. if
+ // the file is not present in the new snapshot (addedFiles) then the file
is considered removed
+ Map<String, RowBackedAction> previousFiles = new HashMap<>();
+ StructType physicalSchema;
+
+ // Check if table exists by checking if _delta_log directory exists
+ boolean tableExists = checkTableExists(table);
+
+ if (tableExists) {
+ Snapshot snapshot = table.getLatestSnapshot(engine);
+
+ // Reuse DeltaKernelDataFileExtractor to iterate through existing files
+ // This avoids duplicating the scan logic for reading Delta files
+ try (DataFileIterator fileIterator =
+ dataFileExtractor.iterator(snapshot, table, engine, tableSchema)) {
+
+ while (fileIterator.hasNext()) {
+ InternalDataFile internalFile = fileIterator.next();
+
+ // Convert InternalDataFile back to AddFile to create RemoveFile
action
+ AddFile addFile = createAddFileFromInternalDataFile(internalFile,
snapshot.getSchema());
+ RemoveFile removeFile =
+ new RemoveFile(addFile.toRemoveFileRow(false,
Optional.of(snapshot.getVersion())));
+ String fullPath =
+
DeltaKernelActionsConverter.getFullPathToFile(removeFile.getPath(), table);
+ previousFiles.put(fullPath, removeFile);
+ }
+ } catch (Exception e) {
+ throw new ReadException("Failed to scan existing Delta files", e);
+ }
+
+ physicalSchema = snapshot.getSchema();
+
+ } else {
+
+ // Table doesn't exist yet - no previous files to remove
+ // Convert InternalSchema to StructType for physical schema
+ DeltaKernelSchemaExtractor schemaExtractor =
DeltaKernelSchemaExtractor.getInstance();
+ physicalSchema = schemaExtractor.fromInternalSchema(tableSchema);
+ }
+
+ FilesDiff<InternalFile, RowBackedAction> diff =
+ InternalFilesDiff.findNewAndRemovedFiles(partitionedDataFiles,
previousFiles);
+
+ return applyDiff(
+ diff.getFilesAdded(),
+ diff.getFilesRemoved(),
+ tableSchema,
+ table.getPath(engine),
Review Comment:
Should this be set to `basePath`?
##########
xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelDataFileUpdatesExtractor.java:
##########
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import scala.collection.JavaConverters;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.types.IntegerType;
+import io.delta.kernel.types.StringType;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.model.schema.PartitionTransformType;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+
+public class TestDeltaKernelDataFileUpdatesExtractor {
+
+ @TempDir private Path tempDir;
+
+ private Engine engine;
+ private DeltaKernelDataFileUpdatesExtractor extractor;
+ private InternalSchema testSchema;
+ private StructType physicalSchema;
+
+ @BeforeEach
+ public void setup() {
+ Configuration hadoopConf = new Configuration();
+ engine = DefaultEngine.create(hadoopConf);
+
+ // Create test schema
+ testSchema =
+ InternalSchema.builder()
+ .name("record")
+ .dataType(InternalType.RECORD)
+ .fields(
+ Arrays.asList(
+ InternalField.builder()
+ .name("id")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("name")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .isNullable(true)
+ .build())
+ .build()))
+ .build();
+
+ // Create physical schema
+ physicalSchema =
+ new StructType()
+ .add(new StructField("id", IntegerType.INTEGER, false))
+ .add(new StructField("name", StringType.STRING, true));
+
+ // Initialize extractor
+ extractor =
+ DeltaKernelDataFileUpdatesExtractor.builder()
+ .engine(engine)
+ .basePath(tempDir.toString())
+ .includeColumnStats(false)
+ .build();
+ }
+
+ @Test
+ public void testCreateAddFileAction() throws IOException {
+ // Create a test data file
+ String testFilePath = tempDir.resolve("test_data.parquet").toString();
+ Files.createFile(Paths.get(testFilePath));
+
+ InternalDataFile dataFile =
+ InternalDataFile.builder()
+ .physicalPath(testFilePath)
+ .fileSizeBytes(1024L)
+ .lastModified(Instant.now().toEpochMilli())
+ .recordCount(100L)
+ .partitionValues(Collections.emptyList())
+ .columnStats(Collections.emptyList())
+ .build();
+
+ // Create a simple Delta table for testing
+ Table table = createSimpleDeltaTable();
+
+ List<PartitionFileGroup> partitionedDataFiles =
+ Collections.singletonList(
+ PartitionFileGroup.builder()
+ .files(Collections.singletonList(dataFile))
+ .partitionValues(Collections.emptyList())
+ .build());
+
+ // Execute applySnapshot
+ scala.collection.Seq<RowBackedAction> actions =
+ extractor.applySnapshot(table, partitionedDataFiles, testSchema);
+
+ // Verify actions are created
+ assertNotNull(actions);
+ List<RowBackedAction> actionList = JavaConverters.seqAsJavaList(actions);
+ assertFalse(actionList.isEmpty(), "Should have at least one action");
+
+ // Verify we have AddFile actions
+ boolean hasAddFile = actionList.stream().anyMatch(action -> action
instanceof AddFile);
+ assertTrue(hasAddFile, "Should contain AddFile actions");
Review Comment:
Can we assert on the content of the `AddFile` to make sure it is aligned
with our expectations?
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelConversionTarget.java:
##########
@@ -0,0 +1,483 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import lombok.Getter;
+import lombok.extern.log4j.Log4j2;
+
+import org.apache.hadoop.conf.Configuration;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import io.delta.kernel.Operation;
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.Transaction;
+import io.delta.kernel.TransactionBuilder;
+import io.delta.kernel.TransactionCommitResult;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.exceptions.TableNotFoundException;
+import io.delta.kernel.hook.PostCommitHook;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.Metadata;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterable;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.conversion.TargetTable;
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.exception.UpdateException;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.metadata.TableSyncMetadata;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.model.storage.TableFormat;
+import org.apache.xtable.spi.sync.ConversionTarget;
+
+/**
+ * Implementation of {@link ConversionTarget} for Delta Lake using the Delta
Kernel API.
+ *
+ * <p>This implementation uses Delta Kernel (io.delta.kernel) instead of Delta
Standalone for write
+ * operations, providing better compatibility with cloud storage (S3, GCS,
Azure Blob Storage, HDFS)
+ * and improved support for Delta Lake 3.x features.
+ *
+ * <p><strong>Initialization:</strong> This class supports two initialization
patterns:
+ *
+ * <ul>
+ * <li><strong>Factory/ServiceLoader Pattern:</strong> Use the no-arg
constructor followed by
+ * {@link #init(TargetTable, Configuration)}. This is used by {@link
+ * org.apache.xtable.conversion.ConversionTargetFactory}.
+ * <li><strong>Direct Testing Pattern:</strong> Use the constructor with
{@link TargetTable} and
+ * {@link Engine} parameters for direct instantiation with custom
dependencies in tests.
+ * </ul>
+ *
+ * <p><strong>Important:</strong> Do not mix initialization patterns. If you
use the parameterized
+ * constructor, do not call {@link #init(TargetTable, Configuration)}
afterward, as it will
+ * overwrite the custom Engine.
+ *
+ * <p><strong>Exception Handling:</strong> This implementation only catches
{@link
+ * io.delta.kernel.exceptions.TableNotFoundException} when checking for table
existence, allowing
+ * other exceptions (network errors, permission issues, corrupted metadata) to
propagate rather than
+ * being silently masked. This ensures real errors are visible and fail fast.
+ *
+ * <p><strong>Known Limitations:</strong>
+ *
+ * <ul>
+ * <li><strong>Commit Tags:</strong> Delta Kernel 4.0.0 does not support
commit tags in commitInfo
+ * (e.g., XTABLE_METADATA tags). This affects source-to-target commit
identifier mapping.
+ * <li><strong>Schema Evolution:</strong> Schema changes are handled through
Delta Kernel's
+ * transaction API, which may have different semantics compared to Delta
Standalone.
+ * <li><strong>Internal API Usage:</strong> This implementation casts to
internal classes
+ * (SnapshotImpl, TableImpl) to access metadata and commit history, as
Delta Kernel 4.0.0
+ * lacks public APIs for these operations. These casts are brittle and
may break on version
+ * upgrades. Public API alternatives should be used when available.
+ * </ul>
+ *
+ * <p><strong>Implementation Choice:</strong> Delta Kernel API was chosen over
Delta Standalone to:
+ *
+ * <ul>
+ * <li>Support newer Delta Lake features and protocol versions
+ * <li>Align with the Delta Lake community's direction (Kernel is the
recommended API)
+ * <li>Reduce dependency on Spark-specific implementations
+ * </ul>
+ *
+ * @see ConversionTarget
+ * @see io.delta.kernel.Table
+ * @see io.delta.kernel.Transaction
+ */
+@Log4j2
+public class DeltaKernelConversionTarget implements ConversionTarget {
+ private static final String DELTA_LOG_RETENTION_DURATION =
"delta.logRetentionDuration";
+
+ private DeltaKernelSchemaExtractor schemaExtractor;
+ private DeltaKernelPartitionExtractor partitionExtractor;
+ private DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor;
+
+ private String basePath;
+ private long logRetentionInHours;
+ private DeltaKernelConversionTarget.TransactionState transactionState;
+ private Engine engine;
+
+ /**
+ * No-arg constructor for ServiceLoader instantiation. Must call {@link
#init(TargetTable,
+ * Configuration)} before use.
+ */
+ public DeltaKernelConversionTarget() {}
+
+ /**
+ * Creates a fully initialized DeltaKernelConversionTarget with custom
Engine. Typically used in
+ * tests. Do not call {@link #init(TargetTable, Configuration)} after this.
+ *
+ * @param targetTable the target table configuration
+ * @param engine custom Delta Kernel engine instance
+ */
+ public DeltaKernelConversionTarget(TargetTable targetTable, Engine engine) {
+ this(
+ targetTable.getBasePath(),
+ targetTable.getMetadataRetention().toHours(),
+ engine,
+ DeltaKernelSchemaExtractor.getInstance(),
+ DeltaKernelPartitionExtractor.getInstance(),
+ DeltaKernelDataFileUpdatesExtractor.builder()
+ .engine(engine)
+ .basePath(targetTable.getBasePath())
+ // Column statistics are not needed for conversion operations
+ .includeColumnStats(false)
+ .build());
+ }
+
+ @VisibleForTesting
+ DeltaKernelConversionTarget(
+ String tableDataPath,
+ long logRetentionInHours,
+ Engine engine,
+ DeltaKernelSchemaExtractor schemaExtractor,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
+ _init(
+ tableDataPath,
+ logRetentionInHours,
+ engine,
+ schemaExtractor,
+ partitionExtractor,
+ dataKernelFileUpdatesExtractor);
+ }
+
+ /**
+ * Private initialization helper to avoid code duplication between
constructor and init() paths.
+ */
+ private void _init(
+ String tableDataPath,
+ long logRetentionInHours,
+ Engine engine,
+ DeltaKernelSchemaExtractor schemaExtractor,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
+ this.basePath = tableDataPath;
+ this.schemaExtractor = schemaExtractor;
+ this.partitionExtractor = partitionExtractor;
+ this.dataKernelFileUpdatesExtractor = dataKernelFileUpdatesExtractor;
+ this.engine = engine;
+ this.logRetentionInHours = logRetentionInHours;
+ }
+
+ @Override
+ public void init(TargetTable targetTable, Configuration configuration) {
+ Engine engine = DefaultEngine.create(configuration);
+
+ _init(
+ targetTable.getBasePath(),
+ targetTable.getMetadataRetention().toHours(),
+ engine,
+ DeltaKernelSchemaExtractor.getInstance(),
+ DeltaKernelPartitionExtractor.getInstance(),
+ DeltaKernelDataFileUpdatesExtractor.builder()
+ .engine(engine)
+ .basePath(targetTable.getBasePath())
+ // Column statistics are not needed for conversion operations
+ .includeColumnStats(false)
+ .build());
+ }
+
+ @Override
+ public void beginSync(InternalTable table) {
+ this.transactionState =
+ new DeltaKernelConversionTarget.TransactionState(engine,
logRetentionInHours);
+ }
+
+ @Override
+ public void syncSchema(InternalSchema schema) {
+ transactionState.setLatestSchema(schema);
+ }
+
+ @Override
+ public void syncPartitionSpec(List<InternalPartitionField> partitionSpec) {
+ if (partitionSpec != null) {
+ Map<String, StructField> spec =
+ partitionExtractor.convertToDeltaPartitionFormat(partitionSpec);
+ for (Map.Entry<String, StructField> partitionEntry : spec.entrySet()) {
+ String partitionColumnName = partitionEntry.getKey();
+ StructField partitionField = partitionEntry.getValue();
+
+ transactionState.addPartitionColumn(partitionColumnName);
+ if (partitionField != null
+ && transactionState.getLatestSchema().fields().stream()
+ .noneMatch(field ->
field.getName().equals(partitionField.getName()))) {
+ // add generated columns to schema.
+ transactionState.addColumn(partitionField);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void syncMetadata(TableSyncMetadata metadata) {
+ transactionState.setMetadata(metadata);
+ }
+
+ @Override
+ public void syncFilesForSnapshot(List<PartitionFileGroup>
partitionedDataFiles) {
+ Table table = Table.forPath(engine, basePath);
+ transactionState.setActions(
+ dataKernelFileUpdatesExtractor.applySnapshot(
+ table, partitionedDataFiles,
transactionState.getLatestSchemaInternal()));
+ }
+
+ @Override
+ public void syncFilesForDiff(InternalFilesDiff internalFilesDiff) {
+ Table table = Table.forPath(engine, basePath);
+ transactionState.setActions(
+ dataKernelFileUpdatesExtractor.applyDiff(
+ internalFilesDiff,
+ transactionState.getLatestSchemaInternal(),
+ table.getPath(engine),
+ table.getLatestSnapshot(engine).getSchema()));
+ }
+
+ @Override
+ public void completeSync() {
+ transactionState.commitTransaction();
+ transactionState = null;
+ }
+
+ @Override
+ public Optional<TableSyncMetadata> getTableMetadata() {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getLatestSnapshot(engine);
+
+ // WORKAROUND: Cast to SnapshotImpl (internal class) to access metadata
configuration.
+ // Delta Kernel 4.0.0 does not provide a public API to access table
metadata/configuration.
+ // This cast is brittle and may break on Kernel version upgrades.
+ Metadata metadata = ((SnapshotImpl) snapshot).getMetadata();
+
+ // Get configuration from metadata
+ Map<String, String> configuration = metadata.getConfiguration();
+ String metadataJson = configuration.get(TableSyncMetadata.XTABLE_METADATA);
+
+ return TableSyncMetadata.fromJson(metadataJson);
+ }
+
+ @Override
+ public String getTableFormat() {
+ return TableFormat.DELTA;
+ }
+
+ @Override
+ public Optional<String> getTargetCommitIdentifier(String sourceIdentifier) {
+ // Delta Kernel 4.0.0 does not support commit tags in commitInfo, which
are required for
Review Comment:
Let's make sure there is an issue in XTable tracking this as well.
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelUtils.java:
##########
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import lombok.experimental.UtilityClass;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.exceptions.TableNotFoundException;
+
+/**
+ * Utility methods for working with Delta Kernel API.
+ *
+ * <p>This class provides common helper methods used across Delta Kernel
integration components to
+ * avoid code duplication and ensure consistent behavior.
+ */
+@UtilityClass
+public class DeltaKernelUtils {
+
+ /**
+ * Checks if a Delta table exists at the specified path.
+ *
+ * <p>This method only catches {@link TableNotFoundException}, allowing
other exceptions (network
+ * errors, permission issues, corrupted metadata) to propagate. This ensures
real errors are
+ * visible rather than being silently masked.
+ *
+ * @param engine the Delta Kernel engine to use
+ * @param basePath the path to the Delta table
+ * @return true if the table exists, false if it doesn't exist
+ * @throws RuntimeException if there's an error other than table not found
(e.g., network issues,
+ * permissions)
+ */
+ public static boolean tableExists(Engine engine, String basePath) {
+ try {
+ Table table = Table.forPath(engine, basePath);
+ table.getLatestSnapshot(engine);
Review Comment:
This looks like it will load the snapshot, is there a more lightweight way
to do this?
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelConversionTarget.java:
##########
@@ -0,0 +1,483 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import lombok.Getter;
+import lombok.extern.log4j.Log4j2;
+
+import org.apache.hadoop.conf.Configuration;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import io.delta.kernel.Operation;
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.Transaction;
+import io.delta.kernel.TransactionBuilder;
+import io.delta.kernel.TransactionCommitResult;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.exceptions.TableNotFoundException;
+import io.delta.kernel.hook.PostCommitHook;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.Metadata;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterable;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.conversion.TargetTable;
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.exception.UpdateException;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.metadata.TableSyncMetadata;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.model.storage.TableFormat;
+import org.apache.xtable.spi.sync.ConversionTarget;
+
+/**
+ * Implementation of {@link ConversionTarget} for Delta Lake using the Delta
Kernel API.
+ *
+ * <p>This implementation uses Delta Kernel (io.delta.kernel) instead of Delta
Standalone for write
+ * operations, providing better compatibility with cloud storage (S3, GCS,
Azure Blob Storage, HDFS)
+ * and improved support for Delta Lake 3.x features.
+ *
+ * <p><strong>Initialization:</strong> This class supports two initialization
patterns:
+ *
+ * <ul>
+ * <li><strong>Factory/ServiceLoader Pattern:</strong> Use the no-arg
constructor followed by
+ * {@link #init(TargetTable, Configuration)}. This is used by {@link
+ * org.apache.xtable.conversion.ConversionTargetFactory}.
+ * <li><strong>Direct Testing Pattern:</strong> Use the constructor with
{@link TargetTable} and
+ * {@link Engine} parameters for direct instantiation with custom
dependencies in tests.
+ * </ul>
+ *
+ * <p><strong>Important:</strong> Do not mix initialization patterns. If you
use the parameterized
+ * constructor, do not call {@link #init(TargetTable, Configuration)}
afterward, as it will
+ * overwrite the custom Engine.
+ *
+ * <p><strong>Exception Handling:</strong> This implementation only catches
{@link
+ * io.delta.kernel.exceptions.TableNotFoundException} when checking for table
existence, allowing
+ * other exceptions (network errors, permission issues, corrupted metadata) to
propagate rather than
+ * being silently masked. This ensures real errors are visible and fail fast.
+ *
+ * <p><strong>Known Limitations:</strong>
+ *
+ * <ul>
+ * <li><strong>Commit Tags:</strong> Delta Kernel 4.0.0 does not support
commit tags in commitInfo
+ * (e.g., XTABLE_METADATA tags). This affects source-to-target commit
identifier mapping.
+ * <li><strong>Schema Evolution:</strong> Schema changes are handled through
Delta Kernel's
+ * transaction API, which may have different semantics compared to Delta
Standalone.
+ * <li><strong>Internal API Usage:</strong> This implementation casts to
internal classes
+ * (SnapshotImpl, TableImpl) to access metadata and commit history, as
Delta Kernel 4.0.0
+ * lacks public APIs for these operations. These casts are brittle and
may break on version
+ * upgrades. Public API alternatives should be used when available.
+ * </ul>
+ *
+ * <p><strong>Implementation Choice:</strong> Delta Kernel API was chosen over
Delta Standalone to:
+ *
+ * <ul>
+ * <li>Support newer Delta Lake features and protocol versions
+ * <li>Align with the Delta Lake community's direction (Kernel is the
recommended API)
+ * <li>Reduce dependency on Spark-specific implementations
+ * </ul>
+ *
+ * @see ConversionTarget
+ * @see io.delta.kernel.Table
+ * @see io.delta.kernel.Transaction
+ */
+@Log4j2
+public class DeltaKernelConversionTarget implements ConversionTarget {
+ private static final String DELTA_LOG_RETENTION_DURATION =
"delta.logRetentionDuration";
+
+ private DeltaKernelSchemaExtractor schemaExtractor;
+ private DeltaKernelPartitionExtractor partitionExtractor;
+ private DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor;
+
+ private String basePath;
+ private long logRetentionInHours;
+ private DeltaKernelConversionTarget.TransactionState transactionState;
+ private Engine engine;
+
+ /**
+ * No-arg constructor for ServiceLoader instantiation. Must call {@link
#init(TargetTable,
+ * Configuration)} before use.
+ */
+ public DeltaKernelConversionTarget() {}
+
+ /**
+ * Creates a fully initialized DeltaKernelConversionTarget with custom
Engine. Typically used in
+ * tests. Do not call {@link #init(TargetTable, Configuration)} after this.
+ *
+ * @param targetTable the target table configuration
+ * @param engine custom Delta Kernel engine instance
+ */
+ public DeltaKernelConversionTarget(TargetTable targetTable, Engine engine) {
+ this(
+ targetTable.getBasePath(),
+ targetTable.getMetadataRetention().toHours(),
+ engine,
+ DeltaKernelSchemaExtractor.getInstance(),
+ DeltaKernelPartitionExtractor.getInstance(),
+ DeltaKernelDataFileUpdatesExtractor.builder()
+ .engine(engine)
+ .basePath(targetTable.getBasePath())
+ // Column statistics are not needed for conversion operations
+ .includeColumnStats(false)
+ .build());
+ }
+
+ @VisibleForTesting
+ DeltaKernelConversionTarget(
+ String tableDataPath,
+ long logRetentionInHours,
+ Engine engine,
+ DeltaKernelSchemaExtractor schemaExtractor,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
+ _init(
+ tableDataPath,
+ logRetentionInHours,
+ engine,
+ schemaExtractor,
+ partitionExtractor,
+ dataKernelFileUpdatesExtractor);
+ }
+
+ /**
+ * Private initialization helper to avoid code duplication between
constructor and init() paths.
+ */
+ private void _init(
+ String tableDataPath,
+ long logRetentionInHours,
+ Engine engine,
+ DeltaKernelSchemaExtractor schemaExtractor,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
+ this.basePath = tableDataPath;
+ this.schemaExtractor = schemaExtractor;
+ this.partitionExtractor = partitionExtractor;
+ this.dataKernelFileUpdatesExtractor = dataKernelFileUpdatesExtractor;
+ this.engine = engine;
+ this.logRetentionInHours = logRetentionInHours;
+ }
+
+ @Override
+ public void init(TargetTable targetTable, Configuration configuration) {
+ Engine engine = DefaultEngine.create(configuration);
+
+ _init(
+ targetTable.getBasePath(),
+ targetTable.getMetadataRetention().toHours(),
+ engine,
+ DeltaKernelSchemaExtractor.getInstance(),
+ DeltaKernelPartitionExtractor.getInstance(),
+ DeltaKernelDataFileUpdatesExtractor.builder()
+ .engine(engine)
+ .basePath(targetTable.getBasePath())
+ // Column statistics are not needed for conversion operations
+ .includeColumnStats(false)
+ .build());
+ }
+
+ @Override
+ public void beginSync(InternalTable table) {
+ this.transactionState =
+ new DeltaKernelConversionTarget.TransactionState(engine,
logRetentionInHours);
+ }
+
+ @Override
+ public void syncSchema(InternalSchema schema) {
+ transactionState.setLatestSchema(schema);
+ }
+
+ @Override
+ public void syncPartitionSpec(List<InternalPartitionField> partitionSpec) {
+ if (partitionSpec != null) {
+ Map<String, StructField> spec =
+ partitionExtractor.convertToDeltaPartitionFormat(partitionSpec);
+ for (Map.Entry<String, StructField> partitionEntry : spec.entrySet()) {
+ String partitionColumnName = partitionEntry.getKey();
+ StructField partitionField = partitionEntry.getValue();
+
+ transactionState.addPartitionColumn(partitionColumnName);
+ if (partitionField != null
+ && transactionState.getLatestSchema().fields().stream()
+ .noneMatch(field ->
field.getName().equals(partitionField.getName()))) {
+ // add generated columns to schema.
+ transactionState.addColumn(partitionField);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void syncMetadata(TableSyncMetadata metadata) {
+ transactionState.setMetadata(metadata);
+ }
+
+ @Override
+ public void syncFilesForSnapshot(List<PartitionFileGroup>
partitionedDataFiles) {
+ Table table = Table.forPath(engine, basePath);
+ transactionState.setActions(
+ dataKernelFileUpdatesExtractor.applySnapshot(
+ table, partitionedDataFiles,
transactionState.getLatestSchemaInternal()));
+ }
+
+ @Override
+ public void syncFilesForDiff(InternalFilesDiff internalFilesDiff) {
+ Table table = Table.forPath(engine, basePath);
+ transactionState.setActions(
+ dataKernelFileUpdatesExtractor.applyDiff(
+ internalFilesDiff,
+ transactionState.getLatestSchemaInternal(),
+ table.getPath(engine),
+ table.getLatestSnapshot(engine).getSchema()));
+ }
+
+ @Override
+ public void completeSync() {
+ transactionState.commitTransaction();
+ transactionState = null;
+ }
+
+ @Override
+ public Optional<TableSyncMetadata> getTableMetadata() {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getLatestSnapshot(engine);
+
+ // WORKAROUND: Cast to SnapshotImpl (internal class) to access metadata
configuration.
+ // Delta Kernel 4.0.0 does not provide a public API to access table
metadata/configuration.
+ // This cast is brittle and may break on Kernel version upgrades.
+ Metadata metadata = ((SnapshotImpl) snapshot).getMetadata();
+
+ // Get configuration from metadata
+ Map<String, String> configuration = metadata.getConfiguration();
+ String metadataJson = configuration.get(TableSyncMetadata.XTABLE_METADATA);
+
+ return TableSyncMetadata.fromJson(metadataJson);
+ }
+
+ @Override
+ public String getTableFormat() {
+ return TableFormat.DELTA;
+ }
+
+ @Override
+ public Optional<String> getTargetCommitIdentifier(String sourceIdentifier) {
+ // Delta Kernel 4.0.0 does not support commit tags in commitInfo, which
are required for
+ // source-to-target commit identifier mapping. This limitation is
documented in:
+ // https://github.com/delta-io/delta/issues/6167
+ //
+ // Unlike DeltaConversionTarget (which uses Delta Standalone with commit
tag support),
+ // DeltaKernelConversionTarget cannot retrieve commit tags from Delta
Kernel's API.
+ // Rather than silently scanning all commits (O(n) performance cost) and
always returning
+ // empty, we explicitly throw an exception to indicate this feature is
unsupported.
+ //
+ // When Delta Kernel adds commit tag support, this method can be
reimplemented to:
+ // 1. Scan commit history using tableImpl.getChanges(engine, 0,
currentVersion, actionSet)
+ // 2. Extract tags from CommitInfo.tags MapValue
+ // 3. Parse XTABLE_METADATA from tags and match sourceIdentifier
+ throw new NotSupportedException(
+ "Source-to-target commit identifier mapping is not supported in
DeltaKernelConversionTarget. "
+ + "Delta Kernel 4.0.0 does not support commit tags in commitInfo. "
+ + "See: https://github.com/delta-io/delta/issues/6167");
+ }
+
+ private class TransactionState {
+ private final Engine engine;
+ private final long retentionInHours;
+ private final List<String> partitionColumns;
+ @Getter private StructType latestSchema;
+ @Getter private InternalSchema latestSchemaInternal;
+ private TableSyncMetadata metadata;
+ private List<RowBackedAction> actions;
+
+ private TransactionState(Engine engine, long retentionInHours) {
+ this.engine = engine;
+ this.partitionColumns = new ArrayList<>();
+ this.retentionInHours = retentionInHours;
+
+ try {
+ Table table = Table.forPath(engine, basePath);
+ this.latestSchema = table.getLatestSnapshot(engine).getSchema();
+ } catch (TableNotFoundException e) {
+ // Expected: table doesn't exist yet on first sync
+ this.latestSchema = null;
+ }
+ // Let other exceptions propagate (network issues, permissions,
corrupted metadata, etc.)
+ }
+
+ /**
+ * Adds a partition column name to the list. Package-private to allow
access from outer class.
+ */
+ void addPartitionColumn(String columnName) {
+ partitionColumns.add(columnName);
+ }
+
+ void setMetadata(TableSyncMetadata metadata) {
+ this.metadata = metadata;
+ }
+
+ /**
+ * Sets the actions to be committed. Converts from Scala Seq to Java List
for internal storage.
+ */
+ void setActions(Seq<RowBackedAction> scalaActions) {
+ this.actions = JavaConverters.seqAsJavaList(scalaActions);
+ }
+
+ private void addColumn(StructField field) {
+ latestSchema = latestSchema.add(field);
+ latestSchemaInternal = schemaExtractor.toInternalSchema(latestSchema);
+ }
+
+ private void setLatestSchema(InternalSchema schema) {
+ this.latestSchemaInternal = schema;
+ this.latestSchema = schemaExtractor.fromInternalSchema(schema);
+ }
+
+ private void commitTransaction() {
+ boolean tableExists = checkTableExists();
+
+ Operation operation = tableExists ? Operation.WRITE :
Operation.CREATE_TABLE;
+
+ if (!tableExists) {
+ File tableDir = new File(basePath);
+ if (!tableDir.exists()) {
+ tableDir.mkdirs();
+ }
+ }
+
+ Table table = Table.forPath(engine, basePath);
+ TransactionBuilder txnBuilder =
+ table.createTransactionBuilder(engine, "XTable Delta Sync",
operation);
+
+ // Schema evolution for existing tables is handled via Metadata actions
manually
+ // as Delta Kernel 4.0.0 doesn't support schema evolution via withSchema
+ if (!tableExists) {
+ txnBuilder = txnBuilder.withSchema(engine, latestSchema);
+
+ if (!partitionColumns.isEmpty()) {
+ txnBuilder = txnBuilder.withPartitionColumns(engine,
partitionColumns);
+ }
+ }
+
+ Map<String, String> tableProperties = getConfigurationsForDeltaSync();
+ txnBuilder = txnBuilder.withTableProperties(engine, tableProperties);
+
+ Transaction txn = txnBuilder.build(engine);
+ List<Row> allActionRows = new ArrayList<>();
+
+ // Iterate through actions (Java List) and convert to Row format
+ for (RowBackedAction action : actions) {
+
+ if (action instanceof AddFile) {
+ AddFile addFile = (AddFile) action;
+ Row wrappedRow =
+
io.delta.kernel.internal.actions.SingleAction.createAddFileSingleAction(
Review Comment:
Let's start using imports throughout the PR please. Do a sanity check of the
files and make sure you are using them throughout. Highlighting every line
leads to a lot of noise on the reviews.
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileUpdatesExtractor.java:
##########
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import lombok.Builder;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.MapValue;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.util.VectorUtils;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.exception.ReadException;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FilesDiff;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalFile;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.paths.PathUtils;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+@Builder
+public class DeltaKernelDataFileUpdatesExtractor {
+ @Builder.Default
+ private final DeltaKernelStatsExtractor deltaStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor deltaKernelPartitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelDataFileExtractor dataFileExtractor =
+ DeltaKernelDataFileExtractor.builder().build();
+
+ private final Engine engine;
+ private final String basePath;
+ private final boolean includeColumnStats;
+
+ public Seq<RowBackedAction> applySnapshot(
+ Table table, List<PartitionFileGroup> partitionedDataFiles,
InternalSchema tableSchema) {
+
+ // all files in the current delta snapshot are potential candidates for
remove actions, i.e. if
+ // the file is not present in the new snapshot (addedFiles) then the file
is considered removed
+ Map<String, RowBackedAction> previousFiles = new HashMap<>();
+ StructType physicalSchema;
+
+ // Check if table exists by checking if _delta_log directory exists
+ boolean tableExists = checkTableExists(table);
+
+ if (tableExists) {
+ Snapshot snapshot = table.getLatestSnapshot(engine);
+
+ // Reuse DeltaKernelDataFileExtractor to iterate through existing files
+ // This avoids duplicating the scan logic for reading Delta files
+ try (DataFileIterator fileIterator =
+ dataFileExtractor.iterator(snapshot, table, engine, tableSchema)) {
+
+ while (fileIterator.hasNext()) {
+ InternalDataFile internalFile = fileIterator.next();
+
+ // Convert InternalDataFile back to AddFile to create RemoveFile
action
+ AddFile addFile = createAddFileFromInternalDataFile(internalFile,
snapshot.getSchema());
+ RemoveFile removeFile =
+ new RemoveFile(addFile.toRemoveFileRow(false,
Optional.of(snapshot.getVersion())));
+ String fullPath =
+
DeltaKernelActionsConverter.getFullPathToFile(removeFile.getPath(), table);
+ previousFiles.put(fullPath, removeFile);
+ }
+ } catch (Exception e) {
+ throw new ReadException("Failed to scan existing Delta files", e);
+ }
+
+ physicalSchema = snapshot.getSchema();
+
+ } else {
+
+ // Table doesn't exist yet - no previous files to remove
+ // Convert InternalSchema to StructType for physical schema
+ DeltaKernelSchemaExtractor schemaExtractor =
DeltaKernelSchemaExtractor.getInstance();
+ physicalSchema = schemaExtractor.fromInternalSchema(tableSchema);
+ }
+
+ FilesDiff<InternalFile, RowBackedAction> diff =
+ InternalFilesDiff.findNewAndRemovedFiles(partitionedDataFiles,
previousFiles);
+
+ return applyDiff(
+ diff.getFilesAdded(),
+ diff.getFilesRemoved(),
+ tableSchema,
+ table.getPath(engine),
+ physicalSchema);
+ }
+
+ private boolean checkTableExists(Table table) {
+ return DeltaKernelUtils.tableExists(engine, table.getPath(engine));
+ }
+
+ /**
+ * Converts an InternalDataFile back to Delta Kernel's AddFile action. This
is needed to create
+ * RemoveFile actions from existing files.
+ */
+ private AddFile createAddFileFromInternalDataFile(
Review Comment:
This is very similar to `createAddFileAction` - can we consolidate?
##########
xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelSchemaExtractor.java:
##########
@@ -873,4 +883,142 @@ public void testIcebergToDeltaUUIDSupport() {
internalSchema,
DeltaKernelSchemaExtractor.getInstance().toInternalSchema(structRepresentation));
}
+
+ // ========== Tests for fromInternalSchema() - New Tests ==========
+
+ @Test
Review Comment:
Let's have tests with nested fields, lists and maps as well
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileUpdatesExtractor.java:
##########
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import lombok.Builder;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.MapValue;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.util.VectorUtils;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.exception.ReadException;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FilesDiff;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalFile;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.paths.PathUtils;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+@Builder
+public class DeltaKernelDataFileUpdatesExtractor {
+ @Builder.Default
+ private final DeltaKernelStatsExtractor deltaStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor deltaKernelPartitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelDataFileExtractor dataFileExtractor =
+ DeltaKernelDataFileExtractor.builder().build();
+
+ private final Engine engine;
+ private final String basePath;
+ private final boolean includeColumnStats;
+
+ public Seq<RowBackedAction> applySnapshot(
+ Table table, List<PartitionFileGroup> partitionedDataFiles,
InternalSchema tableSchema) {
+
+ // all files in the current delta snapshot are potential candidates for
remove actions, i.e. if
+ // the file is not present in the new snapshot (addedFiles) then the file
is considered removed
+ Map<String, RowBackedAction> previousFiles = new HashMap<>();
+ StructType physicalSchema;
+
+ // Check if table exists by checking if _delta_log directory exists
+ boolean tableExists = checkTableExists(table);
+
+ if (tableExists) {
+ Snapshot snapshot = table.getLatestSnapshot(engine);
+
+ // Reuse DeltaKernelDataFileExtractor to iterate through existing files
+ // This avoids duplicating the scan logic for reading Delta files
+ try (DataFileIterator fileIterator =
+ dataFileExtractor.iterator(snapshot, table, engine, tableSchema)) {
+
+ while (fileIterator.hasNext()) {
+ InternalDataFile internalFile = fileIterator.next();
+
+ // Convert InternalDataFile back to AddFile to create RemoveFile
action
+ AddFile addFile = createAddFileFromInternalDataFile(internalFile,
snapshot.getSchema());
+ RemoveFile removeFile =
+ new RemoveFile(addFile.toRemoveFileRow(false,
Optional.of(snapshot.getVersion())));
+ String fullPath =
+
DeltaKernelActionsConverter.getFullPathToFile(removeFile.getPath(), table);
+ previousFiles.put(fullPath, removeFile);
+ }
+ } catch (Exception e) {
+ throw new ReadException("Failed to scan existing Delta files", e);
+ }
+
+ physicalSchema = snapshot.getSchema();
+
+ } else {
+
+ // Table doesn't exist yet - no previous files to remove
+ // Convert InternalSchema to StructType for physical schema
+ DeltaKernelSchemaExtractor schemaExtractor =
DeltaKernelSchemaExtractor.getInstance();
+ physicalSchema = schemaExtractor.fromInternalSchema(tableSchema);
+ }
+
+ FilesDiff<InternalFile, RowBackedAction> diff =
+ InternalFilesDiff.findNewAndRemovedFiles(partitionedDataFiles,
previousFiles);
+
+ return applyDiff(
+ diff.getFilesAdded(),
+ diff.getFilesRemoved(),
+ tableSchema,
+ table.getPath(engine),
+ physicalSchema);
+ }
+
+ private boolean checkTableExists(Table table) {
+ return DeltaKernelUtils.tableExists(engine, table.getPath(engine));
+ }
+
+ /**
+ * Converts an InternalDataFile back to Delta Kernel's AddFile action. This
is needed to create
+ * RemoveFile actions from existing files.
+ */
+ private AddFile createAddFileFromInternalDataFile(
+ InternalDataFile internalFile, StructType physicalSchema) {
+ // Extract partition values from InternalDataFile using existing logic
+ Map<String, String> partitionValuesMap =
+
deltaKernelPartitionExtractor.partitionValueSerialization(internalFile);
+ MapValue partitionValues = convertToMapValue(partitionValuesMap);
+
+ // Create AddFile Row using the same pattern as createAddFileAction
+ Row addFileRow =
+ AddFile.createAddFileRow(
+ physicalSchema,
+ PathUtils.getRelativePath(internalFile.getPhysicalPath(),
basePath),
+ partitionValues,
+ internalFile.getFileSizeBytes(),
+ internalFile.getLastModified(),
+ true, // dataChange - assume true for existing files
+ Optional.empty(), // deletionVector
+ Optional.empty(), // tags
+ Optional.empty(), // baseRowId
+ Optional.empty(), // defaultRowCommitVersion
+ Optional.empty()); // stats - set to empty since we're creating
RemoveFile
+
+ // Wrap the Row back into an AddFile object
+ return new AddFile(addFileRow);
+ }
+
+ public Seq<RowBackedAction> applyDiff(
+ InternalFilesDiff internalFilesDiff,
+ InternalSchema tableSchema,
+ String tableBasePath,
+ StructType physicalSchema) {
+ List<RowBackedAction> removeActions =
+ internalFilesDiff.dataFilesRemoved().stream()
+ .map(dFile -> createAddFileAction(dFile, tableBasePath,
physicalSchema))
+ .map(addFile -> new RemoveFile(addFile.toRemoveFileRow(false,
Optional.empty())))
+
.collect(CustomCollectors.toList(internalFilesDiff.dataFilesRemoved().size()));
+ return applyDiff(
+ internalFilesDiff.dataFilesAdded(),
+ removeActions,
+ tableSchema,
+ tableBasePath,
+ physicalSchema);
+ }
+
+ private Seq<RowBackedAction> applyDiff(
+ Set<? extends InternalFile> filesAdded,
+ Collection<RowBackedAction> removeFileActions,
+ InternalSchema tableSchema,
Review Comment:
`tableSchema` is unused, can it be removed from the method signature?
##########
xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelDataFileUpdatesExtractor.java:
##########
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import scala.collection.JavaConverters;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.types.IntegerType;
+import io.delta.kernel.types.StringType;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.model.schema.PartitionTransformType;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+
+public class TestDeltaKernelDataFileUpdatesExtractor {
+
+ @TempDir private Path tempDir;
+
+ private Engine engine;
+ private DeltaKernelDataFileUpdatesExtractor extractor;
+ private InternalSchema testSchema;
+ private StructType physicalSchema;
+
+ @BeforeEach
+ public void setup() {
+ Configuration hadoopConf = new Configuration();
+ engine = DefaultEngine.create(hadoopConf);
+
+ // Create test schema
+ testSchema =
+ InternalSchema.builder()
+ .name("record")
+ .dataType(InternalType.RECORD)
+ .fields(
+ Arrays.asList(
+ InternalField.builder()
+ .name("id")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("name")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .isNullable(true)
+ .build())
+ .build()))
+ .build();
+
+ // Create physical schema
+ physicalSchema =
+ new StructType()
+ .add(new StructField("id", IntegerType.INTEGER, false))
+ .add(new StructField("name", StringType.STRING, true));
+
+ // Initialize extractor
+ extractor =
+ DeltaKernelDataFileUpdatesExtractor.builder()
+ .engine(engine)
+ .basePath(tempDir.toString())
+ .includeColumnStats(false)
+ .build();
+ }
+
+ @Test
+ public void testCreateAddFileAction() throws IOException {
+ // Create a test data file
+ String testFilePath = tempDir.resolve("test_data.parquet").toString();
+ Files.createFile(Paths.get(testFilePath));
+
+ InternalDataFile dataFile =
+ InternalDataFile.builder()
+ .physicalPath(testFilePath)
+ .fileSizeBytes(1024L)
+ .lastModified(Instant.now().toEpochMilli())
+ .recordCount(100L)
+ .partitionValues(Collections.emptyList())
+ .columnStats(Collections.emptyList())
+ .build();
+
+ // Create a simple Delta table for testing
+ Table table = createSimpleDeltaTable();
+
+ List<PartitionFileGroup> partitionedDataFiles =
+ Collections.singletonList(
+ PartitionFileGroup.builder()
+ .files(Collections.singletonList(dataFile))
+ .partitionValues(Collections.emptyList())
+ .build());
+
+ // Execute applySnapshot
+ scala.collection.Seq<RowBackedAction> actions =
+ extractor.applySnapshot(table, partitionedDataFiles, testSchema);
+
+ // Verify actions are created
+ assertNotNull(actions);
+ List<RowBackedAction> actionList = JavaConverters.seqAsJavaList(actions);
+ assertFalse(actionList.isEmpty(), "Should have at least one action");
+
+ // Verify we have AddFile actions
+ boolean hasAddFile = actionList.stream().anyMatch(action -> action
instanceof AddFile);
+ assertTrue(hasAddFile, "Should contain AddFile actions");
+ }
+
+ @Test
+ public void testApplySnapshotWithPartitionedData() throws IOException {
+ // Create test data files with partitions
+ String testFilePath1 =
tempDir.resolve("partition1/test_data1.parquet").toString();
+ String testFilePath2 =
tempDir.resolve("partition2/test_data2.parquet").toString();
+ Files.createDirectories(Paths.get(testFilePath1).getParent());
+ Files.createDirectories(Paths.get(testFilePath2).getParent());
+ Files.createFile(Paths.get(testFilePath1));
+ Files.createFile(Paths.get(testFilePath2));
+
+ InternalPartitionField partitionField =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("partition_col")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build();
+
+ PartitionValue partitionValue1 =
+ PartitionValue.builder()
+ .partitionField(partitionField)
+ .range(Range.scalar("partition1"))
+ .build();
+
+ PartitionValue partitionValue2 =
+ PartitionValue.builder()
+ .partitionField(partitionField)
+ .range(Range.scalar("partition2"))
+ .build();
+
+ InternalDataFile dataFile1 =
+ InternalDataFile.builder()
+ .physicalPath(testFilePath1)
+ .fileSizeBytes(1024L)
+ .lastModified(Instant.now().toEpochMilli())
+ .recordCount(50L)
+ .partitionValues(Collections.singletonList(partitionValue1))
+ .columnStats(Collections.emptyList())
+ .build();
+
+ InternalDataFile dataFile2 =
+ InternalDataFile.builder()
+ .physicalPath(testFilePath2)
+ .fileSizeBytes(2048L)
+ .lastModified(Instant.now().toEpochMilli())
+ .recordCount(75L)
+ .partitionValues(Collections.singletonList(partitionValue2))
+ .columnStats(Collections.emptyList())
+ .build();
+
+ Table table = createSimpleDeltaTable();
+
+ List<PartitionFileGroup> partitionedDataFiles =
+ Arrays.asList(
+ PartitionFileGroup.builder()
+ .files(Collections.singletonList(dataFile1))
+ .partitionValues(Collections.singletonList(partitionValue1))
+ .build(),
+ PartitionFileGroup.builder()
+ .files(Collections.singletonList(dataFile2))
+ .partitionValues(Collections.singletonList(partitionValue2))
+ .build());
+
+ // Execute applySnapshot
+ scala.collection.Seq<RowBackedAction> actions =
+ extractor.applySnapshot(table, partitionedDataFiles, testSchema);
+
+ // Verify
+ assertNotNull(actions);
+ List<RowBackedAction> actionList = JavaConverters.seqAsJavaList(actions);
+ assertFalse(actionList.isEmpty(), "Should have actions for partitioned
data");
+
+ // Should have AddFile actions for new files
+ long addFileCount = actionList.stream().filter(action -> action instanceof
AddFile).count();
+ assertTrue(addFileCount >= 2, "Should have at least 2 AddFile actions");
+ }
+
+ @Test
+ public void testDifferentialSyncWithExistingData() throws IOException {
+ // This test simulates a real differential sync scenario:
+ // 1. Delta table has existing files: file1.parquet, file2.parquet
+ // 2. New sync brings: file2.parquet (unchanged), file3.parquet (new)
+ // 3. Expected result: AddFile for file3, RemoveFile for file1
+
+ // Step 1: Create a Delta table with existing data
+ Path tablePath = tempDir.resolve("delta_table_with_data");
+ Files.createDirectories(tablePath);
+ Path deltaLogPath = tablePath.resolve("_delta_log");
+ Files.createDirectories(deltaLogPath);
+
+ // Create existing data files
+ Path existingFile1 = tablePath.resolve("file1.parquet");
+ Path existingFile2 = tablePath.resolve("file2.parquet");
+ Files.createFile(existingFile1);
+ Files.createFile(existingFile2);
+
+ // Create initial commit with file1 and file2
+ Path initialCommit = deltaLogPath.resolve("00000000000000000000.json");
+ String initialCommitJson =
+ "{\"protocol\":{\"minReaderVersion\":1,\"minWriterVersion\":2}}\n"
+ +
"{\"metaData\":{\"id\":\"test-id\",\"format\":{\"provider\":\"parquet\",\"options\":{}},\"schemaString\":\""
+ + physicalSchema.toJson().replace("\"", "\\\"")
+ +
"\",\"partitionColumns\":[],\"configuration\":{},\"createdTime\":"
+ + System.currentTimeMillis()
+ + "}}\n"
+ +
"{\"add\":{\"path\":\"file1.parquet\",\"partitionValues\":{},\"size\":1024,\"modificationTime\":"
+ + Instant.now().toEpochMilli()
+ + ",\"dataChange\":true,\"stats\":\"{}\"}}\n"
+ +
"{\"add\":{\"path\":\"file2.parquet\",\"partitionValues\":{},\"size\":2048,\"modificationTime\":"
+ + Instant.now().toEpochMilli()
+ + ",\"dataChange\":true,\"stats\":\"{}\"}}\n";
+ Files.write(initialCommit,
initialCommitJson.getBytes(StandardCharsets.UTF_8));
+
+ // Create the table
+ Table table = Table.forPath(engine, tablePath.toString());
+ assertNotNull(table);
+
+ // Step 2: Prepare new sync data - file2 (unchanged) + file3 (new)
+ Path newFile3 = tablePath.resolve("file3.parquet");
+ Files.createFile(newFile3);
+
+ InternalDataFile dataFile2 =
+ InternalDataFile.builder()
+ .physicalPath(existingFile2.toString())
+ .fileSizeBytes(2048L)
+ .lastModified(Instant.now().toEpochMilli())
+ .recordCount(100L)
+ .partitionValues(Collections.emptyList())
+ .columnStats(Collections.emptyList())
+ .build();
+
+ InternalDataFile dataFile3 =
+ InternalDataFile.builder()
+ .physicalPath(newFile3.toString())
+ .fileSizeBytes(3072L)
+ .lastModified(Instant.now().toEpochMilli())
+ .recordCount(150L)
+ .partitionValues(Collections.emptyList())
+ .columnStats(Collections.emptyList())
+ .build();
+
+ List<PartitionFileGroup> newPartitionedDataFiles =
+ Collections.singletonList(
+ PartitionFileGroup.builder()
+ .files(Arrays.asList(dataFile2, dataFile3))
+ .partitionValues(Collections.emptyList())
+ .build());
+
+ // Step 3: Apply snapshot (differential sync)
+ DeltaKernelDataFileUpdatesExtractor syncExtractor =
+ DeltaKernelDataFileUpdatesExtractor.builder()
+ .engine(engine)
+ .basePath(tablePath.toString())
+ .includeColumnStats(false)
+ .build();
+
+ scala.collection.Seq<RowBackedAction> actions =
+ syncExtractor.applySnapshot(table, newPartitionedDataFiles,
testSchema);
+
+ // Step 4: Verify the differential sync results
+ assertNotNull(actions, "Actions should not be null");
+ List<RowBackedAction> actionList = JavaConverters.seqAsJavaList(actions);
+ assertFalse(actionList.isEmpty(), "Should have actions for differential
sync");
+
+ // Count AddFile and RemoveFile actions
+ long addFileCount = actionList.stream().filter(action -> action instanceof
AddFile).count();
+ long removeFileCount =
+ actionList.stream().filter(action -> action instanceof
RemoveFile).count();
+
+ // Verify: Should have AddFile for file3 (new file)
+ assertTrue(addFileCount >= 1, "Should have at least 1 AddFile action for
new file (file3)");
Review Comment:
These counts should be strict. Only 1 file is expected to be added and 1
removed
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelConversionTarget.java:
##########
@@ -0,0 +1,483 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import lombok.Getter;
+import lombok.extern.log4j.Log4j2;
+
+import org.apache.hadoop.conf.Configuration;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import io.delta.kernel.Operation;
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.Transaction;
+import io.delta.kernel.TransactionBuilder;
+import io.delta.kernel.TransactionCommitResult;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.exceptions.TableNotFoundException;
+import io.delta.kernel.hook.PostCommitHook;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.Metadata;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterable;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.conversion.TargetTable;
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.exception.UpdateException;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.metadata.TableSyncMetadata;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.model.storage.TableFormat;
+import org.apache.xtable.spi.sync.ConversionTarget;
+
+/**
+ * Implementation of {@link ConversionTarget} for Delta Lake using the Delta
Kernel API.
+ *
+ * <p>This implementation uses Delta Kernel (io.delta.kernel) instead of Delta
Standalone for write
+ * operations, providing better compatibility with cloud storage (S3, GCS,
Azure Blob Storage, HDFS)
+ * and improved support for Delta Lake 3.x features.
+ *
+ * <p><strong>Initialization:</strong> This class supports two initialization
patterns:
+ *
+ * <ul>
+ * <li><strong>Factory/ServiceLoader Pattern:</strong> Use the no-arg
constructor followed by
+ * {@link #init(TargetTable, Configuration)}. This is used by {@link
+ * org.apache.xtable.conversion.ConversionTargetFactory}.
+ * <li><strong>Direct Testing Pattern:</strong> Use the constructor with
{@link TargetTable} and
+ * {@link Engine} parameters for direct instantiation with custom
dependencies in tests.
+ * </ul>
+ *
+ * <p><strong>Important:</strong> Do not mix initialization patterns. If you
use the parameterized
+ * constructor, do not call {@link #init(TargetTable, Configuration)}
afterward, as it will
+ * overwrite the custom Engine.
+ *
+ * <p><strong>Exception Handling:</strong> This implementation only catches
{@link
+ * io.delta.kernel.exceptions.TableNotFoundException} when checking for table
existence, allowing
+ * other exceptions (network errors, permission issues, corrupted metadata) to
propagate rather than
+ * being silently masked. This ensures real errors are visible and fail fast.
+ *
+ * <p><strong>Known Limitations:</strong>
+ *
+ * <ul>
+ * <li><strong>Commit Tags:</strong> Delta Kernel 4.0.0 does not support
commit tags in commitInfo
+ * (e.g., XTABLE_METADATA tags). This affects source-to-target commit
identifier mapping.
+ * <li><strong>Schema Evolution:</strong> Schema changes are handled through
Delta Kernel's
+ * transaction API, which may have different semantics compared to Delta
Standalone.
+ * <li><strong>Internal API Usage:</strong> This implementation casts to
internal classes
+ * (SnapshotImpl, TableImpl) to access metadata and commit history, as
Delta Kernel 4.0.0
+ * lacks public APIs for these operations. These casts are brittle and
may break on version
+ * upgrades. Public API alternatives should be used when available.
+ * </ul>
+ *
+ * <p><strong>Implementation Choice:</strong> Delta Kernel API was chosen over
Delta Standalone to:
+ *
+ * <ul>
+ * <li>Support newer Delta Lake features and protocol versions
+ * <li>Align with the Delta Lake community's direction (Kernel is the
recommended API)
+ * <li>Reduce dependency on Spark-specific implementations
+ * </ul>
+ *
+ * @see ConversionTarget
+ * @see io.delta.kernel.Table
+ * @see io.delta.kernel.Transaction
+ */
+@Log4j2
+public class DeltaKernelConversionTarget implements ConversionTarget {
+ private static final String DELTA_LOG_RETENTION_DURATION =
"delta.logRetentionDuration";
+
+ private DeltaKernelSchemaExtractor schemaExtractor;
+ private DeltaKernelPartitionExtractor partitionExtractor;
+ private DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor;
+
+ private String basePath;
+ private long logRetentionInHours;
+ private DeltaKernelConversionTarget.TransactionState transactionState;
+ private Engine engine;
+
+ /**
+ * No-arg constructor for ServiceLoader instantiation. Must call {@link
#init(TargetTable,
+ * Configuration)} before use.
+ */
+ public DeltaKernelConversionTarget() {}
+
+ /**
+ * Creates a fully initialized DeltaKernelConversionTarget with custom
Engine. Typically used in
+ * tests. Do not call {@link #init(TargetTable, Configuration)} after this.
+ *
+ * @param targetTable the target table configuration
+ * @param engine custom Delta Kernel engine instance
+ */
+ public DeltaKernelConversionTarget(TargetTable targetTable, Engine engine) {
+ this(
+ targetTable.getBasePath(),
+ targetTable.getMetadataRetention().toHours(),
+ engine,
+ DeltaKernelSchemaExtractor.getInstance(),
+ DeltaKernelPartitionExtractor.getInstance(),
+ DeltaKernelDataFileUpdatesExtractor.builder()
+ .engine(engine)
+ .basePath(targetTable.getBasePath())
+ // Column statistics are not needed for conversion operations
+ .includeColumnStats(false)
+ .build());
+ }
+
+ @VisibleForTesting
+ DeltaKernelConversionTarget(
+ String tableDataPath,
+ long logRetentionInHours,
+ Engine engine,
+ DeltaKernelSchemaExtractor schemaExtractor,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
+ _init(
+ tableDataPath,
+ logRetentionInHours,
+ engine,
+ schemaExtractor,
+ partitionExtractor,
+ dataKernelFileUpdatesExtractor);
+ }
+
+ /**
+ * Private initialization helper to avoid code duplication between
constructor and init() paths.
+ */
+ private void _init(
+ String tableDataPath,
+ long logRetentionInHours,
+ Engine engine,
+ DeltaKernelSchemaExtractor schemaExtractor,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
+ this.basePath = tableDataPath;
+ this.schemaExtractor = schemaExtractor;
+ this.partitionExtractor = partitionExtractor;
+ this.dataKernelFileUpdatesExtractor = dataKernelFileUpdatesExtractor;
+ this.engine = engine;
+ this.logRetentionInHours = logRetentionInHours;
+ }
+
+ @Override
+ public void init(TargetTable targetTable, Configuration configuration) {
+ Engine engine = DefaultEngine.create(configuration);
+
+ _init(
+ targetTable.getBasePath(),
+ targetTable.getMetadataRetention().toHours(),
+ engine,
+ DeltaKernelSchemaExtractor.getInstance(),
+ DeltaKernelPartitionExtractor.getInstance(),
+ DeltaKernelDataFileUpdatesExtractor.builder()
+ .engine(engine)
+ .basePath(targetTable.getBasePath())
+ // Column statistics are not needed for conversion operations
+ .includeColumnStats(false)
+ .build());
+ }
+
+ @Override
+ public void beginSync(InternalTable table) {
+ this.transactionState =
+ new DeltaKernelConversionTarget.TransactionState(engine,
logRetentionInHours);
+ }
+
+ @Override
+ public void syncSchema(InternalSchema schema) {
+ transactionState.setLatestSchema(schema);
+ }
+
+ @Override
+ public void syncPartitionSpec(List<InternalPartitionField> partitionSpec) {
+ if (partitionSpec != null) {
+ Map<String, StructField> spec =
+ partitionExtractor.convertToDeltaPartitionFormat(partitionSpec);
+ for (Map.Entry<String, StructField> partitionEntry : spec.entrySet()) {
+ String partitionColumnName = partitionEntry.getKey();
+ StructField partitionField = partitionEntry.getValue();
+
+ transactionState.addPartitionColumn(partitionColumnName);
+ if (partitionField != null
+ && transactionState.getLatestSchema().fields().stream()
+ .noneMatch(field ->
field.getName().equals(partitionField.getName()))) {
+ // add generated columns to schema.
+ transactionState.addColumn(partitionField);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void syncMetadata(TableSyncMetadata metadata) {
+ transactionState.setMetadata(metadata);
+ }
+
+ @Override
+ public void syncFilesForSnapshot(List<PartitionFileGroup>
partitionedDataFiles) {
+ Table table = Table.forPath(engine, basePath);
+ transactionState.setActions(
+ dataKernelFileUpdatesExtractor.applySnapshot(
+ table, partitionedDataFiles,
transactionState.getLatestSchemaInternal()));
+ }
+
+ @Override
+ public void syncFilesForDiff(InternalFilesDiff internalFilesDiff) {
+ Table table = Table.forPath(engine, basePath);
+ transactionState.setActions(
+ dataKernelFileUpdatesExtractor.applyDiff(
+ internalFilesDiff,
+ transactionState.getLatestSchemaInternal(),
+ table.getPath(engine),
+ table.getLatestSnapshot(engine).getSchema()));
+ }
+
+ @Override
+ public void completeSync() {
+ transactionState.commitTransaction();
+ transactionState = null;
+ }
+
+ @Override
+ public Optional<TableSyncMetadata> getTableMetadata() {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getLatestSnapshot(engine);
+
+ // WORKAROUND: Cast to SnapshotImpl (internal class) to access metadata
configuration.
+ // Delta Kernel 4.0.0 does not provide a public API to access table
metadata/configuration.
+ // This cast is brittle and may break on Kernel version upgrades.
+ Metadata metadata = ((SnapshotImpl) snapshot).getMetadata();
+
+ // Get configuration from metadata
+ Map<String, String> configuration = metadata.getConfiguration();
+ String metadataJson = configuration.get(TableSyncMetadata.XTABLE_METADATA);
+
+ return TableSyncMetadata.fromJson(metadataJson);
+ }
+
+ @Override
+ public String getTableFormat() {
+ return TableFormat.DELTA;
+ }
+
+ @Override
+ public Optional<String> getTargetCommitIdentifier(String sourceIdentifier) {
+ // Delta Kernel 4.0.0 does not support commit tags in commitInfo, which
are required for
+ // source-to-target commit identifier mapping. This limitation is
documented in:
+ // https://github.com/delta-io/delta/issues/6167
+ //
+ // Unlike DeltaConversionTarget (which uses Delta Standalone with commit
tag support),
+ // DeltaKernelConversionTarget cannot retrieve commit tags from Delta
Kernel's API.
+ // Rather than silently scanning all commits (O(n) performance cost) and
always returning
+ // empty, we explicitly throw an exception to indicate this feature is
unsupported.
+ //
+ // When Delta Kernel adds commit tag support, this method can be
reimplemented to:
+ // 1. Scan commit history using tableImpl.getChanges(engine, 0,
currentVersion, actionSet)
+ // 2. Extract tags from CommitInfo.tags MapValue
+ // 3. Parse XTABLE_METADATA from tags and match sourceIdentifier
+ throw new NotSupportedException(
+ "Source-to-target commit identifier mapping is not supported in
DeltaKernelConversionTarget. "
+ + "Delta Kernel 4.0.0 does not support commit tags in commitInfo. "
+ + "See: https://github.com/delta-io/delta/issues/6167");
+ }
+
+ private class TransactionState {
+ private final Engine engine;
+ private final long retentionInHours;
+ private final List<String> partitionColumns;
+ @Getter private StructType latestSchema;
+ @Getter private InternalSchema latestSchemaInternal;
+ private TableSyncMetadata metadata;
+ private List<RowBackedAction> actions;
+
+ private TransactionState(Engine engine, long retentionInHours) {
+ this.engine = engine;
+ this.partitionColumns = new ArrayList<>();
+ this.retentionInHours = retentionInHours;
+
+ try {
+ Table table = Table.forPath(engine, basePath);
+ this.latestSchema = table.getLatestSnapshot(engine).getSchema();
+ } catch (TableNotFoundException e) {
+ // Expected: table doesn't exist yet on first sync
+ this.latestSchema = null;
+ }
+ // Let other exceptions propagate (network issues, permissions,
corrupted metadata, etc.)
+ }
+
+ /**
+ * Adds a partition column name to the list. Package-private to allow
access from outer class.
+ */
+ void addPartitionColumn(String columnName) {
+ partitionColumns.add(columnName);
+ }
+
+ void setMetadata(TableSyncMetadata metadata) {
+ this.metadata = metadata;
+ }
+
+ /**
+ * Sets the actions to be committed. Converts from Scala Seq to Java List
for internal storage.
+ */
+ void setActions(Seq<RowBackedAction> scalaActions) {
+ this.actions = JavaConverters.seqAsJavaList(scalaActions);
+ }
+
+ private void addColumn(StructField field) {
+ latestSchema = latestSchema.add(field);
+ latestSchemaInternal = schemaExtractor.toInternalSchema(latestSchema);
+ }
+
+ private void setLatestSchema(InternalSchema schema) {
+ this.latestSchemaInternal = schema;
+ this.latestSchema = schemaExtractor.fromInternalSchema(schema);
+ }
+
+ private void commitTransaction() {
+ boolean tableExists = checkTableExists();
+
+ Operation operation = tableExists ? Operation.WRITE :
Operation.CREATE_TABLE;
+
+ if (!tableExists) {
+ File tableDir = new File(basePath);
+ if (!tableDir.exists()) {
+ tableDir.mkdirs();
+ }
+ }
+
+ Table table = Table.forPath(engine, basePath);
+ TransactionBuilder txnBuilder =
+ table.createTransactionBuilder(engine, "XTable Delta Sync",
operation);
+
+ // Schema evolution for existing tables is handled via Metadata actions
manually
Review Comment:
Is this supported as part of this PR?
##########
xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelDataFileUpdatesExtractor.java:
##########
@@ -0,0 +1,417 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Instant;
+import java.util.*;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import scala.collection.JavaConverters;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.types.IntegerType;
+import io.delta.kernel.types.StringType;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.model.schema.PartitionTransformType;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+
+public class TestDeltaKernelDataFileUpdatesExtractor {
+
+ @TempDir private Path tempDir;
+
+ private Engine engine;
+ private DeltaKernelDataFileUpdatesExtractor extractor;
+ private InternalSchema testSchema;
+ private StructType physicalSchema;
+
+ @BeforeEach
+ public void setup() {
+ Configuration hadoopConf = new Configuration();
+ engine = DefaultEngine.create(hadoopConf);
+
+ // Create test schema
+ testSchema =
+ InternalSchema.builder()
+ .name("record")
+ .dataType(InternalType.RECORD)
+ .fields(
+ Arrays.asList(
+ InternalField.builder()
+ .name("id")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("name")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .isNullable(true)
+ .build())
+ .build()))
+ .build();
+
+ // Create physical schema
+ physicalSchema =
+ new StructType()
+ .add(new StructField("id", IntegerType.INTEGER, false))
+ .add(new StructField("name", StringType.STRING, true));
+
+ // Initialize extractor
+ extractor =
+ DeltaKernelDataFileUpdatesExtractor.builder()
+ .engine(engine)
+ .basePath(tempDir.toString())
+ .includeColumnStats(false)
+ .build();
+ }
+
+ @Test
+ public void testCreateAddFileAction() throws IOException {
+ // Create a test data file
+ String testFilePath = tempDir.resolve("test_data.parquet").toString();
+ Files.createFile(Paths.get(testFilePath));
+
+ InternalDataFile dataFile =
+ InternalDataFile.builder()
+ .physicalPath(testFilePath)
+ .fileSizeBytes(1024L)
+ .lastModified(Instant.now().toEpochMilli())
+ .recordCount(100L)
+ .partitionValues(Collections.emptyList())
+ .columnStats(Collections.emptyList())
+ .build();
+
+ // Create a simple Delta table for testing
+ Table table = createSimpleDeltaTable();
+
+ List<PartitionFileGroup> partitionedDataFiles =
+ Collections.singletonList(
+ PartitionFileGroup.builder()
+ .files(Collections.singletonList(dataFile))
+ .partitionValues(Collections.emptyList())
+ .build());
+
+ // Execute applySnapshot
+ scala.collection.Seq<RowBackedAction> actions =
+ extractor.applySnapshot(table, partitionedDataFiles, testSchema);
+
+ // Verify actions are created
+ assertNotNull(actions);
+ List<RowBackedAction> actionList = JavaConverters.seqAsJavaList(actions);
+ assertFalse(actionList.isEmpty(), "Should have at least one action");
+
+ // Verify we have AddFile actions
+ boolean hasAddFile = actionList.stream().anyMatch(action -> action
instanceof AddFile);
+ assertTrue(hasAddFile, "Should contain AddFile actions");
+ }
+
+ @Test
+ public void testApplySnapshotWithPartitionedData() throws IOException {
+ // Create test data files with partitions
+ String testFilePath1 =
tempDir.resolve("partition1/test_data1.parquet").toString();
+ String testFilePath2 =
tempDir.resolve("partition2/test_data2.parquet").toString();
+ Files.createDirectories(Paths.get(testFilePath1).getParent());
+ Files.createDirectories(Paths.get(testFilePath2).getParent());
+ Files.createFile(Paths.get(testFilePath1));
+ Files.createFile(Paths.get(testFilePath2));
+
+ InternalPartitionField partitionField =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("partition_col")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build();
+
+ PartitionValue partitionValue1 =
+ PartitionValue.builder()
+ .partitionField(partitionField)
+ .range(Range.scalar("partition1"))
+ .build();
+
+ PartitionValue partitionValue2 =
+ PartitionValue.builder()
+ .partitionField(partitionField)
+ .range(Range.scalar("partition2"))
+ .build();
+
+ InternalDataFile dataFile1 =
+ InternalDataFile.builder()
+ .physicalPath(testFilePath1)
+ .fileSizeBytes(1024L)
+ .lastModified(Instant.now().toEpochMilli())
+ .recordCount(50L)
+ .partitionValues(Collections.singletonList(partitionValue1))
+ .columnStats(Collections.emptyList())
+ .build();
+
+ InternalDataFile dataFile2 =
+ InternalDataFile.builder()
+ .physicalPath(testFilePath2)
+ .fileSizeBytes(2048L)
+ .lastModified(Instant.now().toEpochMilli())
+ .recordCount(75L)
+ .partitionValues(Collections.singletonList(partitionValue2))
+ .columnStats(Collections.emptyList())
+ .build();
+
+ Table table = createSimpleDeltaTable();
+
+ List<PartitionFileGroup> partitionedDataFiles =
+ Arrays.asList(
+ PartitionFileGroup.builder()
+ .files(Collections.singletonList(dataFile1))
+ .partitionValues(Collections.singletonList(partitionValue1))
+ .build(),
+ PartitionFileGroup.builder()
+ .files(Collections.singletonList(dataFile2))
+ .partitionValues(Collections.singletonList(partitionValue2))
+ .build());
+
+ // Execute applySnapshot
+ scala.collection.Seq<RowBackedAction> actions =
+ extractor.applySnapshot(table, partitionedDataFiles, testSchema);
+
+ // Verify
+ assertNotNull(actions);
+ List<RowBackedAction> actionList = JavaConverters.seqAsJavaList(actions);
+ assertFalse(actionList.isEmpty(), "Should have actions for partitioned
data");
+
+ // Should have AddFile actions for new files
+ long addFileCount = actionList.stream().filter(action -> action instanceof
AddFile).count();
+ assertTrue(addFileCount >= 2, "Should have at least 2 AddFile actions");
+ }
+
+ @Test
+ public void testApplySnapshotWithRemovedFiles() throws IOException {
+ // This test verifies that files in the current snapshot but not in new
data
+ // are converted to RemoveFile actions
+
+ Table table = createSimpleDeltaTable();
+
+ // Provide empty partitioned data files (simulating all files removed)
+ List<PartitionFileGroup> partitionedDataFiles = Collections.emptyList();
+
+ // Execute applySnapshot
+ scala.collection.Seq<RowBackedAction> actions =
+ extractor.applySnapshot(table, partitionedDataFiles, testSchema);
+
+ // Verify
+ assertNotNull(actions);
+ List<RowBackedAction> actionList = JavaConverters.seqAsJavaList(actions);
+
+ // If the table had files, they should be converted to RemoveFile actions
+ // Since we created a simple empty table, this might be empty or have
remove actions
+ // depending on the table state
+ assertNotNull(actionList);
+ }
+
+ @Test
+ public void testDifferentialSyncWithExistingData() throws IOException {
+ // This test simulates a real differential sync scenario:
+ // 1. Delta table has existing files: file1.parquet, file2.parquet
+ // 2. New sync brings: file2.parquet (unchanged), file3.parquet (new)
+ // 3. Expected result: AddFile for file3, RemoveFile for file1
+
+ // Step 1: Create a Delta table with existing data
+ Path tablePath = tempDir.resolve("delta_table_with_data");
+ Files.createDirectories(tablePath);
+ Path deltaLogPath = tablePath.resolve("_delta_log");
+ Files.createDirectories(deltaLogPath);
+
+ // Create existing data files
+ Path existingFile1 = tablePath.resolve("file1.parquet");
+ Path existingFile2 = tablePath.resolve("file2.parquet");
+ Files.createFile(existingFile1);
+ Files.createFile(existingFile2);
+
+ // Create initial commit with file1 and file2
+ Path initialCommit = deltaLogPath.resolve("00000000000000000000.json");
+ String initialCommitJson =
+ "{\"protocol\":{\"minReaderVersion\":1,\"minWriterVersion\":2}}\n"
+ +
"{\"metaData\":{\"id\":\"test-id\",\"format\":{\"provider\":\"parquet\",\"options\":{}},\"schemaString\":\""
+ + physicalSchema.toJson().replace("\"", "\\\"")
+ +
"\",\"partitionColumns\":[],\"configuration\":{},\"createdTime\":"
+ + System.currentTimeMillis()
+ + "}}\n"
+ +
"{\"add\":{\"path\":\"file1.parquet\",\"partitionValues\":{},\"size\":1024,\"modificationTime\":"
+ + Instant.now().toEpochMilli()
+ + ",\"dataChange\":true,\"stats\":\"{}\"}}\n"
+ +
"{\"add\":{\"path\":\"file2.parquet\",\"partitionValues\":{},\"size\":2048,\"modificationTime\":"
+ + Instant.now().toEpochMilli()
+ + ",\"dataChange\":true,\"stats\":\"{}\"}}\n";
+ Files.write(initialCommit,
initialCommitJson.getBytes(StandardCharsets.UTF_8));
+
+ // Create the table
+ Table table = Table.forPath(engine, tablePath.toString());
+ assertNotNull(table);
+
+ // Step 2: Prepare new sync data - file2 (unchanged) + file3 (new)
+ Path newFile3 = tablePath.resolve("file3.parquet");
+ Files.createFile(newFile3);
+
+ InternalDataFile dataFile2 =
+ InternalDataFile.builder()
+ .physicalPath(existingFile2.toString())
+ .fileSizeBytes(2048L)
+ .lastModified(Instant.now().toEpochMilli())
+ .recordCount(100L)
+ .partitionValues(Collections.emptyList())
+ .columnStats(Collections.emptyList())
+ .build();
+
+ InternalDataFile dataFile3 =
+ InternalDataFile.builder()
+ .physicalPath(newFile3.toString())
+ .fileSizeBytes(3072L)
+ .lastModified(Instant.now().toEpochMilli())
+ .recordCount(150L)
+ .partitionValues(Collections.emptyList())
+ .columnStats(Collections.emptyList())
+ .build();
+
+ List<PartitionFileGroup> newPartitionedDataFiles =
+ Collections.singletonList(
+ PartitionFileGroup.builder()
+ .files(Arrays.asList(dataFile2, dataFile3))
+ .partitionValues(Collections.emptyList())
+ .build());
+
+ // Step 3: Apply snapshot (differential sync)
+ DeltaKernelDataFileUpdatesExtractor syncExtractor =
+ DeltaKernelDataFileUpdatesExtractor.builder()
+ .engine(engine)
+ .basePath(tablePath.toString())
+ .includeColumnStats(false)
+ .build();
+
+ scala.collection.Seq<RowBackedAction> actions =
+ syncExtractor.applySnapshot(table, newPartitionedDataFiles,
testSchema);
+
+ // Step 4: Verify the differential sync results
+ assertNotNull(actions, "Actions should not be null");
+ List<RowBackedAction> actionList = JavaConverters.seqAsJavaList(actions);
+ assertFalse(actionList.isEmpty(), "Should have actions for differential
sync");
+
+ // Count AddFile and RemoveFile actions
+ long addFileCount = actionList.stream().filter(action -> action instanceof
AddFile).count();
+ long removeFileCount =
+ actionList.stream().filter(action -> action instanceof
RemoveFile).count();
+
+ // Verify: Should have AddFile for file3 (new file)
+ assertTrue(addFileCount >= 1, "Should have at least 1 AddFile action for
new file (file3)");
+
+ // Verify: Should have RemoveFile for file1 (removed from new sync)
+ assertTrue(
+ removeFileCount >= 1,
+ "Should have at least 1 RemoveFile action for file1 that's not in new
sync");
+
+ // Verify specific files in actions
+ boolean hasFile3Add =
+ actionList.stream()
+ .filter(action -> action instanceof AddFile)
+ .map(action -> (AddFile) action)
+ .anyMatch(addFile -> addFile.getPath().contains("file3.parquet"));
+
+ assertTrue(hasFile3Add, "Should have AddFile action for file3.parquet");
+
+ // Note: file2 should not appear in actions as it's unchanged
+ // file1 should appear as RemoveFile as it's not in the new sync
+ System.out.println(
Review Comment:
@vaibhavk1992 can you address this? We don't want printing in our tests
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]