vinishjail97 commented on code in PR #801:
URL: https://github.com/apache/incubator-xtable/pull/801#discussion_r3190469837
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java:
##########
@@ -229,4 +230,118 @@ private InternalSchema toInternalSchema(
.fields(fields)
.build();
}
+
+ /**
+ * Converts an InternalSchema to Delta Kernel StructType.
+ *
+ * @param internalSchema the internal schema representation
+ * @return Delta Kernel StructType
+ */
+ public StructType fromInternalSchema(InternalSchema internalSchema) {
+ List<StructField> fields =
+ internalSchema.getFields().stream()
+ .map(
+ field ->
+ new StructField(
+ field.getName(),
+ convertFieldType(field),
+ field.getSchema().isNullable(),
+ getFieldMetadata(field.getSchema())))
+
.collect(CustomCollectors.toList(internalSchema.getFields().size()));
+ return new StructType(fields);
+ }
+
+ /**
+ * Converts an InternalField to Delta Kernel DataType.
+ *
+ * @param field the internal field
+ * @return Delta Kernel DataType
+ */
+ private DataType convertFieldType(InternalField field) {
+ switch (field.getSchema().getDataType()) {
+ case STRING:
+ case ENUM:
+ return StringType.STRING;
+ case INT:
+ return IntegerType.INTEGER;
+ case LONG:
+ return LongType.LONG;
+ case BYTES:
+ case FIXED:
+ case UUID:
+ return BinaryType.BINARY;
+ case BOOLEAN:
+ return BooleanType.BOOLEAN;
+ case FLOAT:
+ return FloatType.FLOAT;
+ case DATE:
+ return DateType.DATE;
+ case TIMESTAMP:
+ return TimestampType.TIMESTAMP;
+ case TIMESTAMP_NTZ:
+ return TimestampNTZType.TIMESTAMP_NTZ;
+ case DOUBLE:
+ return DoubleType.DOUBLE;
+ case DECIMAL:
+ int precision =
+ (int)
field.getSchema().getMetadata().get(InternalSchema.MetadataKey.DECIMAL_PRECISION);
Review Comment:
`(int) field.getSchema().getMetadata().get(...)` will NPE if metadata is
null, or if the `DECIMAL_PRECISION` / `DECIMAL_SCALE` keys are missing. Use
`Preconditions.checkNotNull(..., "DECIMAL precision/scale required")` so the
failure is actionable instead of a raw NPE.
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java:
##########
@@ -229,4 +230,118 @@ private InternalSchema toInternalSchema(
.fields(fields)
.build();
}
+
+ /**
+ * Converts an InternalSchema to Delta Kernel StructType.
+ *
+ * @param internalSchema the internal schema representation
+ * @return Delta Kernel StructType
+ */
+ public StructType fromInternalSchema(InternalSchema internalSchema) {
+ List<StructField> fields =
+ internalSchema.getFields().stream()
+ .map(
+ field ->
+ new StructField(
+ field.getName(),
+ convertFieldType(field),
+ field.getSchema().isNullable(),
+ getFieldMetadata(field.getSchema())))
+
.collect(CustomCollectors.toList(internalSchema.getFields().size()));
Review Comment:
`fromInternalSchema` doesn't propagate field IDs (`delta.columnMapping.id`)
into `FieldMetadata`. The reverse direction reads them on line 137.
Round-tripping a schema through `toInternalSchema -> fromInternalSchema`
silently drops column-mapping IDs, which would break Delta tables that rely on
column mapping.
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaStatsUtils.java:
##########
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import lombok.AllArgsConstructor;
+import lombok.Value;
+import lombok.extern.log4j.Log4j2;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.fasterxml.jackson.annotation.JsonAnySetter;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.model.exception.ParseException;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.FileStats;
+import org.apache.xtable.model.stat.Range;
+
+/**
+ * Shared utility for Delta stats conversion between XTable internal format
and Delta format. Used
+ * by both Delta Standalone and Delta Kernel implementations.
+ */
+@Log4j2
+public class DeltaStatsUtils {
+ private static final Set<InternalType> FIELD_TYPES_WITH_STATS_SUPPORT =
+ new HashSet<>(
+ Arrays.asList(
+ InternalType.BOOLEAN,
+ InternalType.DATE,
+ InternalType.DECIMAL,
+ InternalType.DOUBLE,
+ InternalType.INT,
+ InternalType.LONG,
+ InternalType.FLOAT,
+ InternalType.STRING,
+ InternalType.TIMESTAMP,
+ InternalType.TIMESTAMP_NTZ));
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ /* this data structure collects type names of all unrecognized Delta Lake
stats. For instance
+ data file stats in presence of delete vectors would contain 'tightBounds'
stat which is
+ currently not handled by XTable */
+ private static final Set<String> unsupportedStats = new HashSet<>();
+
+ private DeltaStatsUtils() {
+ // Utility class, prevent instantiation
+ }
+
+ /**
+ * Converts XTable internal column statistics to Delta format JSON string.
+ *
+ * @param schema the table schema
+ * @param numRecords the number of records in the file
+ * @param columnStats the list of column statistics
+ * @return JSON string in Delta stats format
+ * @throws JsonProcessingException if serialization fails
+ */
+ public static String convertStatsToDeltaFormat(
+ InternalSchema schema, long numRecords, List<ColumnStat> columnStats)
+ throws JsonProcessingException {
+ DeltaStats.DeltaStatsBuilder deltaStatsBuilder = DeltaStats.builder();
+ deltaStatsBuilder.numRecords(numRecords);
+ if (columnStats == null) {
+ return MAPPER.writeValueAsString(deltaStatsBuilder.build());
+ }
+ Set<String> validPaths = getPathsFromStructSchemaForMinAndMaxStats(schema);
+ List<ColumnStat> validColumnStats =
+ columnStats.stream()
+ .filter(stat -> validPaths.contains(stat.getField().getPath()))
+ .collect(Collectors.toList());
+ DeltaStats deltaStats =
+ deltaStatsBuilder
+ .minValues(getMinValues(validColumnStats))
+ .maxValues(getMaxValues(validColumnStats))
+ .nullCount(getNullCount(validColumnStats))
+ .build();
+ return MAPPER.writeValueAsString(deltaStats);
+ }
+
+ /**
+ * Parses Delta stats JSON and extracts column statistics for the specified
fields.
+ *
+ * @param statsJson the Delta stats JSON string
+ * @param fields the fields to extract stats for
+ * @return FileStats containing column statistics and record count
+ * @throws ParseException if JSON parsing fails
+ */
+ public static FileStats parseColumnStatsFromJson(String statsJson,
List<InternalField> fields) {
+ if (StringUtils.isEmpty(statsJson)) {
+ return
FileStats.builder().columnStats(Collections.emptyList()).numRecords(0).build();
+ }
+
+ try {
+ DeltaStats deltaStats = MAPPER.readValue(statsJson, DeltaStats.class);
+
+ collectUnsupportedStats(deltaStats.getAdditionalStats());
+
+ Map<String, Object> fieldPathToMaxValue =
flattenStatMap(deltaStats.getMaxValues());
+ Map<String, Object> fieldPathToMinValue =
flattenStatMap(deltaStats.getMinValues());
+ Map<String, Object> fieldPathToNullCount =
flattenStatMap(deltaStats.getNullCount());
+
+ List<ColumnStat> columnStats =
+ fields.stream()
+ .filter(field ->
fieldPathToMaxValue.containsKey(field.getPath()))
+ .map(
+ field -> {
+ String fieldPath = field.getPath();
+ Object minRaw = fieldPathToMinValue.get(fieldPath);
+ Object maxRaw = fieldPathToMaxValue.get(fieldPath);
+ Object nullCountRaw = fieldPathToNullCount.get(fieldPath);
+
+ Object minValue =
+ minRaw != null
+ ?
DeltaValueConverter.convertFromDeltaColumnStatValue(
+ minRaw, field.getSchema())
+ : null;
+ Object maxValue =
+ maxRaw != null
+ ?
DeltaValueConverter.convertFromDeltaColumnStatValue(
+ maxRaw, field.getSchema())
+ : null;
+ long nullCount =
+ nullCountRaw instanceof Number ? ((Number)
nullCountRaw).longValue() : 0;
+
+ Range range = Range.vector(minValue, maxValue);
+ return ColumnStat.builder()
+ .field(field)
+ .numValues(deltaStats.getNumRecords())
+ .numNulls(nullCount)
+ .range(range)
+ .build();
+ })
+ .collect(CustomCollectors.toList(fields.size()));
+
+ return FileStats.builder()
+ .columnStats(columnStats)
+ .numRecords(deltaStats.getNumRecords())
+ .build();
+ } catch (IOException ex) {
+ throw new ParseException("Unable to parse stats json", ex);
+ }
+ }
+
+ private static Set<String>
getPathsFromStructSchemaForMinAndMaxStats(InternalSchema schema) {
+ return schema.getAllFields().stream()
+ .filter(
+ field -> {
+ InternalType type = field.getSchema().getDataType();
+ return FIELD_TYPES_WITH_STATS_SUPPORT.contains(type);
+ })
+ .map(InternalField::getPath)
+ .collect(Collectors.toSet());
+ }
+
+ private static Map<String, Object> getMinValues(List<ColumnStat>
validColumnStats) {
+ return getValues(validColumnStats, columnStat ->
columnStat.getRange().getMinValue());
+ }
+
+ private static Map<String, Object> getMaxValues(List<ColumnStat>
validColumnStats) {
+ return getValues(validColumnStats, columnStat ->
columnStat.getRange().getMaxValue());
+ }
+
+ private static Map<String, Object> getValues(
+ List<ColumnStat> validColumnStats, Function<ColumnStat, Object>
valueExtractor) {
+ Map<String, Object> jsonObject = new HashMap<>();
+ validColumnStats.forEach(
+ columnStat -> {
+ InternalField field = columnStat.getField();
+ String[] pathParts = field.getPathParts();
+ insertValueAtPath(
+ jsonObject,
+ pathParts,
+ DeltaValueConverter.convertToDeltaColumnStatValue(
+ valueExtractor.apply(columnStat), field.getSchema()));
+ });
+ return jsonObject;
+ }
+
+ private static Map<String, Object> getNullCount(List<ColumnStat>
validColumnStats) {
+ // TODO: Additional work needed to track nulls maps & arrays.
+ Map<String, Object> jsonObject = new HashMap<>();
+ validColumnStats.forEach(
+ columnStat -> {
+ String[] pathParts = columnStat.getField().getPathParts();
+ insertValueAtPath(jsonObject, pathParts, columnStat.getNumNulls());
+ });
+ return jsonObject;
+ }
+
+ private static void insertValueAtPath(
+ Map<String, Object> jsonObject, String[] pathParts, Object value) {
+ if (pathParts == null || pathParts.length == 0) {
+ return;
+ }
+ Map<String, Object> currObject = jsonObject;
+ for (int i = 0; i < pathParts.length; i++) {
+ String part = pathParts[i];
+ if (i == pathParts.length - 1) {
+ currObject.put(part, value);
+ } else {
+ if (!currObject.containsKey(part)) {
+ currObject.put(part, new HashMap<String, Object>());
+ }
+ try {
+ currObject = (HashMap<String, Object>) currObject.get(part);
+ } catch (ClassCastException e) {
+ throw new RuntimeException(
Review Comment:
`throw new RuntimeException(...)` for the cast failure — please use a domain
exception. `ParseException` (already imported on line 47) fits since this is in
the stats-parsing path, and callers can distinguish it from a generic runtime
failure.
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaStatsUtils.java:
##########
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import lombok.AllArgsConstructor;
+import lombok.Value;
+import lombok.extern.log4j.Log4j2;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.fasterxml.jackson.annotation.JsonAnySetter;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.model.exception.ParseException;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.FileStats;
+import org.apache.xtable.model.stat.Range;
+
+/**
+ * Shared utility for Delta stats conversion between XTable internal format
and Delta format. Used
+ * by both Delta Standalone and Delta Kernel implementations.
+ */
+@Log4j2
+public class DeltaStatsUtils {
+ private static final Set<InternalType> FIELD_TYPES_WITH_STATS_SUPPORT =
+ new HashSet<>(
+ Arrays.asList(
+ InternalType.BOOLEAN,
+ InternalType.DATE,
+ InternalType.DECIMAL,
+ InternalType.DOUBLE,
+ InternalType.INT,
+ InternalType.LONG,
+ InternalType.FLOAT,
+ InternalType.STRING,
+ InternalType.TIMESTAMP,
+ InternalType.TIMESTAMP_NTZ));
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ /* this data structure collects type names of all unrecognized Delta Lake
stats. For instance
+ data file stats in presence of delete vectors would contain 'tightBounds'
stat which is
+ currently not handled by XTable */
+ private static final Set<String> unsupportedStats = new HashSet<>();
Review Comment:
`private static final Set<String> unsupportedStats = new HashSet<>()` is
mutable static state shared across both Standalone and Kernel extractors. Two
issues: (a) not thread-safe under concurrent stats parsing — `HashSet` mutation
will race; (b) it leaks across tables and across tests in the same JVM, so test
ordering can change observed behavior. Use `ConcurrentHashMap.newKeySet()` and
consider scoping to an instance instead of static.
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileUpdatesExtractor.java:
##########
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import lombok.Builder;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.MapValue;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.expressions.Column;
+import io.delta.kernel.expressions.Literal;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.util.VectorUtils;
+import io.delta.kernel.statistics.DataFileStatistics;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.exception.ReadException;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.FilesDiff;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalFile;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.paths.PathUtils;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+@Builder
+public class DeltaKernelDataFileUpdatesExtractor {
+ @Builder.Default
+ private final DeltaKernelStatsExtractor deltaStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor deltaKernelPartitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelDataFileExtractor dataFileExtractor =
+ DeltaKernelDataFileExtractor.builder().build();
+
+ private final Engine engine;
+ private final String basePath;
+ private final boolean includeColumnStats;
+
+ /**
+ * Applies snapshot changes, loading the snapshot fresh.
+ *
+ * @param table the Delta table
+ * @param partitionedDataFiles the new data files to sync
+ * @param tableSchema the internal schema
+ * @return sequence of Delta actions (AddFile/RemoveFile)
+ */
+ public Seq<RowBackedAction> applySnapshot(
+ Table table, List<PartitionFileGroup> partitionedDataFiles,
InternalSchema tableSchema) {
+ return applySnapshot(table, partitionedDataFiles, tableSchema, null);
Review Comment:
The 3-arg `applySnapshot` overload (no cached snapshot) is only used by
tests — production code (`DeltaKernelConversionTarget.syncFilesForSnapshot`)
always passes the cached snapshot. Either delete this overload or change tests
to exercise the production path so we're testing what actually runs.
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelStatsExtractor.java:
##########
@@ -18,307 +18,72 @@
package org.apache.xtable.kernel;
-import java.io.IOException;
-import java.util.ArrayDeque;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Queue;
import java.util.Set;
-import java.util.function.Function;
-import java.util.stream.Collectors;
import lombok.AccessLevel;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Getter;
import lombok.NoArgsConstructor;
-import lombok.Value;
-import lombok.extern.log4j.Log4j2;
-import org.apache.commons.lang3.StringUtils;
-
-import com.fasterxml.jackson.annotation.JsonAnySetter;
-import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import io.delta.kernel.internal.actions.AddFile;
-import org.apache.xtable.collectors.CustomCollectors;
-import org.apache.xtable.delta.DeltaValueConverter;
-import org.apache.xtable.model.exception.ParseException;
+import org.apache.xtable.delta.DeltaStatsUtils;
import org.apache.xtable.model.schema.InternalField;
import org.apache.xtable.model.schema.InternalSchema;
-import org.apache.xtable.model.schema.InternalType;
import org.apache.xtable.model.stat.ColumnStat;
import org.apache.xtable.model.stat.FileStats;
-import org.apache.xtable.model.stat.Range;
/**
- * DeltaStatsExtractor extracts column stats and also responsible for their
serialization leveraging
- * {@link DeltaValueConverter}.
+ * Delta Kernel stats extractor - delegates to {@link DeltaStatsUtils} for
shared logic.
+ *
+ * @deprecated This class is a thin wrapper around DeltaStatsUtils. Consider
using DeltaStatsUtils
+ * directly.
Review Comment:
Class-level Javadoc says `@deprecated` but there's no `@Deprecated`
annotation on the class — callers won't get any compile-time warning. Either
add `@Deprecated` (and on the methods that delegate) so it surfaces in IDEs, or
drop the deprecation language. Same in `DeltaStatsExtractor.java`.
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelActionsConverter.java:
##########
@@ -50,6 +50,47 @@ public static DeltaKernelActionsConverter getInstance() {
return INSTANCE;
}
+ /**
+ * Converts AddFile to InternalDataFile using cached table base path (most
efficient).
+ *
+ * @param tableBasePath cached table base path from table.getPath(engine)
+ */
+ public InternalDataFile convertAddActionToInternalDataFile(
+ AddFile addFile,
+ String tableBasePath,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ List<InternalField> fields,
+ boolean includeColumnStats,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelStatsExtractor fileStatsExtractor,
+ Map<String, String> partitionValues) {
+ FileStats fileStats = fileStatsExtractor.getColumnStatsForFile(addFile,
fields);
+ List<ColumnStat> columnStats =
+ includeColumnStats ? fileStats.getColumnStats() :
Collections.emptyList();
+ long recordCount = fileStats.getNumRecords();
+
+ Map<String, String> scalaMap = partitionValues;
Review Comment:
nit: `Map<String, String> scalaMap = partitionValues;` is a no-op rename to
a misleading name — the value is already a Java `Map`, not a Scala one. Just
use `partitionValues` directly. Same pattern on lines 110, 135, 160.
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelActionsConverter.java:
##########
@@ -50,6 +50,47 @@ public static DeltaKernelActionsConverter getInstance() {
return INSTANCE;
}
+ /**
+ * Converts AddFile to InternalDataFile using cached table base path (most
efficient).
+ *
+ * @param tableBasePath cached table base path from table.getPath(engine)
+ */
+ public InternalDataFile convertAddActionToInternalDataFile(
+ AddFile addFile,
+ String tableBasePath,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ List<InternalField> fields,
+ boolean includeColumnStats,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelStatsExtractor fileStatsExtractor,
+ Map<String, String> partitionValues) {
+ FileStats fileStats = fileStatsExtractor.getColumnStatsForFile(addFile,
fields);
+ List<ColumnStat> columnStats =
+ includeColumnStats ? fileStats.getColumnStats() :
Collections.emptyList();
+ long recordCount = fileStats.getNumRecords();
+
+ Map<String, String> scalaMap = partitionValues;
+
+ return InternalDataFile.builder()
+ .physicalPath(getFullPathToFile(addFile.getPath(), tableBasePath))
+ .fileFormat(fileFormat)
+ .fileSizeBytes(addFile.getSize())
+ .lastModified(addFile.getModificationTime())
+ .partitionValues(partitionExtractor.partitionValueExtraction(scalaMap,
partitionFields))
+ .columnStats(columnStats)
+ .recordCount(recordCount)
+ .build();
+ }
+
+ /**
+ * Converts AddFile to InternalDataFile (deprecated - inefficient).
+ *
+ * @deprecated Use {@link #convertAddActionToInternalDataFile(AddFile,
String, FileFormat, List,
+ * List, boolean, DeltaKernelPartitionExtractor,
DeltaKernelStatsExtractor, Map)} with cached
+ * tableBasePath instead to avoid creating new Engine per call
+ */
+ @Deprecated
Review Comment:
These three `@Deprecated` overloads (lines 93, 151, 193) are introduced
*and* marked deprecated in the same PR — they're dead code on day one. I
confirmed via grep that no caller uses the `Table`-taking variants. Please
delete them rather than carrying forward deprecated overloads from the start.
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelConversionTarget.java:
##########
@@ -0,0 +1,498 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static io.delta.kernel.utils.CloseableIterable.inMemoryIterable;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import lombok.Getter;
+import lombok.extern.log4j.Log4j2;
+
+import org.apache.hadoop.conf.Configuration;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import io.delta.kernel.Operation;
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.Transaction;
+import io.delta.kernel.TransactionBuilder;
+import io.delta.kernel.TransactionCommitResult;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.exceptions.TableNotFoundException;
+import io.delta.kernel.hook.PostCommitHook;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.Metadata;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.actions.SingleAction;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterable;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.conversion.TargetTable;
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.exception.UpdateException;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.metadata.TableSyncMetadata;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.model.storage.TableFormat;
+import org.apache.xtable.spi.sync.ConversionTarget;
+
+/**
+ * Implementation of {@link ConversionTarget} for Delta Lake using the Delta
Kernel API.
+ *
+ * <p>This implementation uses Delta Kernel (io.delta.kernel) instead of Delta
Standalone for write
+ * operations, providing better compatibility with cloud storage (S3, GCS,
Azure Blob Storage, HDFS)
+ * and improved support for Delta Lake 3.x features.
+ *
+ * <p><strong>Initialization:</strong> This class supports two initialization
patterns:
+ *
+ * <ul>
+ * <li><strong>Factory/ServiceLoader Pattern:</strong> Use the no-arg
constructor followed by
+ * {@link #init(TargetTable, Configuration)}. This is used by {@link
+ * org.apache.xtable.conversion.ConversionTargetFactory}.
+ * <li><strong>Direct Testing Pattern:</strong> Use the constructor with
{@link TargetTable} and
+ * {@link Engine} parameters for direct instantiation with custom
dependencies in tests.
+ * </ul>
+ *
+ * <p><strong>Important:</strong> Do not mix initialization patterns. If you
use the parameterized
+ * constructor, do not call {@link #init(TargetTable, Configuration)}
afterward, as it will
+ * overwrite the custom Engine.
+ *
+ * <p><strong>Exception Handling:</strong> This implementation only catches
{@link
+ * io.delta.kernel.exceptions.TableNotFoundException} when checking for table
existence, allowing
+ * other exceptions (network errors, permission issues, corrupted metadata) to
propagate rather than
+ * being silently masked. This ensures real errors are visible and fail fast.
+ *
+ * <p><strong>Known Limitations:</strong>
+ *
+ * <ul>
+ * <li><strong>Commit Tags:</strong> Delta Kernel 4.0.0 does not support
commit tags in commitInfo
+ * (e.g., XTABLE_METADATA tags). This affects source-to-target commit
identifier mapping.
+ * Tracked in: https://github.com/apache/incubator-xtable/issues/819
+ * <li><strong>Schema Evolution:</strong> Schema changes are handled through
Delta Kernel's
+ * transaction API, which may have different semantics compared to Delta
Standalone.
+ * <li><strong>Internal API Usage:</strong> This implementation casts to
internal classes
+ * (SnapshotImpl, TableImpl) to access metadata and commit history, as
Delta Kernel 4.0.0
+ * lacks public APIs for these operations. These casts are brittle and
may break on version
+ * upgrades. Public API alternatives should be used when available.
+ * </ul>
+ *
+ * <p><strong>Implementation Choice:</strong> Delta Kernel API was chosen over
Delta Standalone to:
+ *
+ * <ul>
+ * <li>Support newer Delta Lake features and protocol versions
+ * <li>Align with the Delta Lake community's direction (Kernel is the
recommended API)
+ * <li>Reduce dependency on Spark-specific implementations
+ * </ul>
+ *
+ * @see ConversionTarget
+ * @see io.delta.kernel.Table
+ * @see io.delta.kernel.Transaction
+ */
+@Log4j2
+public class DeltaKernelConversionTarget implements ConversionTarget {
+ private static final String DELTA_LOG_RETENTION_DURATION =
"delta.logRetentionDuration";
+
+ private DeltaKernelSchemaExtractor schemaExtractor;
+ private DeltaKernelPartitionExtractor partitionExtractor;
+ private DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor;
+
+ private String basePath;
+ private long logRetentionInHours;
+ private DeltaKernelConversionTarget.TransactionState transactionState;
+ private Engine engine;
+
+ /**
+ * No-arg constructor for ServiceLoader instantiation. Must call {@link
#init(TargetTable,
+ * Configuration)} before use.
+ */
+ public DeltaKernelConversionTarget() {}
+
+ /**
+ * Creates a fully initialized DeltaKernelConversionTarget with custom
Engine. Typically used in
+ * tests. Do not call {@link #init(TargetTable, Configuration)} after this.
+ *
+ * @param targetTable the target table configuration
+ * @param engine custom Delta Kernel engine instance
+ */
+ public DeltaKernelConversionTarget(TargetTable targetTable, Engine engine) {
+ this(
+ targetTable.getBasePath(),
+ targetTable.getMetadataRetention().toHours(),
+ engine,
+ DeltaKernelSchemaExtractor.getInstance(),
+ DeltaKernelPartitionExtractor.getInstance(),
+ DeltaKernelDataFileUpdatesExtractor.builder()
+ .engine(engine)
+ .basePath(targetTable.getBasePath())
+ // Column statistics are not needed for conversion operations
+ .includeColumnStats(false)
+ .build());
+ }
+
+ @VisibleForTesting
+ DeltaKernelConversionTarget(
+ String tableDataPath,
+ long logRetentionInHours,
+ Engine engine,
+ DeltaKernelSchemaExtractor schemaExtractor,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
+ _init(
+ tableDataPath,
+ logRetentionInHours,
+ engine,
+ schemaExtractor,
+ partitionExtractor,
+ dataKernelFileUpdatesExtractor);
+ }
+
+ /**
+ * Private initialization helper to avoid code duplication between
constructor and init() paths.
+ */
+ private void _init(
Review Comment:
Method name `_init` starts with `_`, which violates Google Java Style and
will fail `./gradlew spotlessJavaCheck`. Rename to `initInternal` /
`commonInit`, or just overload the existing `init`.
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelConversionTarget.java:
##########
@@ -0,0 +1,498 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static io.delta.kernel.utils.CloseableIterable.inMemoryIterable;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import lombok.Getter;
+import lombok.extern.log4j.Log4j2;
+
+import org.apache.hadoop.conf.Configuration;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import io.delta.kernel.Operation;
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.Transaction;
+import io.delta.kernel.TransactionBuilder;
+import io.delta.kernel.TransactionCommitResult;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.exceptions.TableNotFoundException;
+import io.delta.kernel.hook.PostCommitHook;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.Metadata;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.actions.SingleAction;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterable;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.conversion.TargetTable;
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.exception.UpdateException;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.metadata.TableSyncMetadata;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.model.storage.TableFormat;
+import org.apache.xtable.spi.sync.ConversionTarget;
+
+/**
+ * Implementation of {@link ConversionTarget} for Delta Lake using the Delta
Kernel API.
+ *
+ * <p>This implementation uses Delta Kernel (io.delta.kernel) instead of Delta
Standalone for write
+ * operations, providing better compatibility with cloud storage (S3, GCS,
Azure Blob Storage, HDFS)
+ * and improved support for Delta Lake 3.x features.
+ *
+ * <p><strong>Initialization:</strong> This class supports two initialization
patterns:
+ *
+ * <ul>
+ * <li><strong>Factory/ServiceLoader Pattern:</strong> Use the no-arg
constructor followed by
+ * {@link #init(TargetTable, Configuration)}. This is used by {@link
+ * org.apache.xtable.conversion.ConversionTargetFactory}.
+ * <li><strong>Direct Testing Pattern:</strong> Use the constructor with
{@link TargetTable} and
+ * {@link Engine} parameters for direct instantiation with custom
dependencies in tests.
+ * </ul>
+ *
+ * <p><strong>Important:</strong> Do not mix initialization patterns. If you
use the parameterized
+ * constructor, do not call {@link #init(TargetTable, Configuration)}
afterward, as it will
+ * overwrite the custom Engine.
+ *
+ * <p><strong>Exception Handling:</strong> This implementation only catches
{@link
+ * io.delta.kernel.exceptions.TableNotFoundException} when checking for table
existence, allowing
+ * other exceptions (network errors, permission issues, corrupted metadata) to
propagate rather than
+ * being silently masked. This ensures real errors are visible and fail fast.
+ *
+ * <p><strong>Known Limitations:</strong>
+ *
+ * <ul>
+ * <li><strong>Commit Tags:</strong> Delta Kernel 4.0.0 does not support
commit tags in commitInfo
+ * (e.g., XTABLE_METADATA tags). This affects source-to-target commit
identifier mapping.
+ * Tracked in: https://github.com/apache/incubator-xtable/issues/819
+ * <li><strong>Schema Evolution:</strong> Schema changes are handled through
Delta Kernel's
+ * transaction API, which may have different semantics compared to Delta
Standalone.
+ * <li><strong>Internal API Usage:</strong> This implementation casts to
internal classes
+ * (SnapshotImpl, TableImpl) to access metadata and commit history, as
Delta Kernel 4.0.0
+ * lacks public APIs for these operations. These casts are brittle and
may break on version
+ * upgrades. Public API alternatives should be used when available.
+ * </ul>
+ *
+ * <p><strong>Implementation Choice:</strong> Delta Kernel API was chosen over
Delta Standalone to:
+ *
+ * <ul>
+ * <li>Support newer Delta Lake features and protocol versions
+ * <li>Align with the Delta Lake community's direction (Kernel is the
recommended API)
+ * <li>Reduce dependency on Spark-specific implementations
+ * </ul>
+ *
+ * @see ConversionTarget
+ * @see io.delta.kernel.Table
+ * @see io.delta.kernel.Transaction
+ */
+@Log4j2
+public class DeltaKernelConversionTarget implements ConversionTarget {
+ private static final String DELTA_LOG_RETENTION_DURATION =
"delta.logRetentionDuration";
+
+ private DeltaKernelSchemaExtractor schemaExtractor;
+ private DeltaKernelPartitionExtractor partitionExtractor;
+ private DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor;
+
+ private String basePath;
+ private long logRetentionInHours;
+ private DeltaKernelConversionTarget.TransactionState transactionState;
+ private Engine engine;
+
+ /**
+ * No-arg constructor for ServiceLoader instantiation. Must call {@link
#init(TargetTable,
+ * Configuration)} before use.
+ */
+ public DeltaKernelConversionTarget() {}
+
+ /**
+ * Creates a fully initialized DeltaKernelConversionTarget with custom
Engine. Typically used in
+ * tests. Do not call {@link #init(TargetTable, Configuration)} after this.
+ *
+ * @param targetTable the target table configuration
+ * @param engine custom Delta Kernel engine instance
+ */
+ public DeltaKernelConversionTarget(TargetTable targetTable, Engine engine) {
+ this(
+ targetTable.getBasePath(),
+ targetTable.getMetadataRetention().toHours(),
+ engine,
+ DeltaKernelSchemaExtractor.getInstance(),
+ DeltaKernelPartitionExtractor.getInstance(),
+ DeltaKernelDataFileUpdatesExtractor.builder()
+ .engine(engine)
+ .basePath(targetTable.getBasePath())
+ // Column statistics are not needed for conversion operations
+ .includeColumnStats(false)
+ .build());
+ }
+
+ @VisibleForTesting
+ DeltaKernelConversionTarget(
+ String tableDataPath,
+ long logRetentionInHours,
+ Engine engine,
+ DeltaKernelSchemaExtractor schemaExtractor,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
+ _init(
+ tableDataPath,
+ logRetentionInHours,
+ engine,
+ schemaExtractor,
+ partitionExtractor,
+ dataKernelFileUpdatesExtractor);
+ }
+
+ /**
+ * Private initialization helper to avoid code duplication between
constructor and init() paths.
+ */
+ private void _init(
+ String tableDataPath,
+ long logRetentionInHours,
+ Engine engine,
+ DeltaKernelSchemaExtractor schemaExtractor,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelDataFileUpdatesExtractor dataKernelFileUpdatesExtractor) {
+ this.basePath = tableDataPath;
+ this.schemaExtractor = schemaExtractor;
+ this.partitionExtractor = partitionExtractor;
+ this.dataKernelFileUpdatesExtractor = dataKernelFileUpdatesExtractor;
+ this.engine = engine;
+ this.logRetentionInHours = logRetentionInHours;
+ }
+
+ @Override
+ public void init(TargetTable targetTable, Configuration configuration) {
+ Engine engine = DefaultEngine.create(configuration);
+
+ _init(
+ targetTable.getBasePath(),
+ targetTable.getMetadataRetention().toHours(),
+ engine,
+ DeltaKernelSchemaExtractor.getInstance(),
+ DeltaKernelPartitionExtractor.getInstance(),
+ DeltaKernelDataFileUpdatesExtractor.builder()
+ .engine(engine)
+ .basePath(targetTable.getBasePath())
+ .includeColumnStats(true)
+ .build());
+ }
+
+ @Override
+ public void beginSync(InternalTable table) {
+ this.transactionState =
+ new DeltaKernelConversionTarget.TransactionState(engine,
logRetentionInHours);
+ }
+
+ @Override
+ public void syncSchema(InternalSchema schema) {
+ transactionState.setLatestSchema(schema);
+ }
+
+ @Override
+ public void syncPartitionSpec(List<InternalPartitionField> partitionSpec) {
+ if (partitionSpec != null) {
+ Map<String, StructField> spec =
+ partitionExtractor.convertToDeltaPartitionFormat(partitionSpec);
+ for (Map.Entry<String, StructField> partitionEntry : spec.entrySet()) {
+ String partitionColumnName = partitionEntry.getKey();
+ StructField partitionField = partitionEntry.getValue();
+
+ transactionState.addPartitionColumn(partitionColumnName);
+ if (partitionField != null
+ && transactionState.getLatestSchema().fields().stream()
+ .noneMatch(field ->
field.getName().equals(partitionField.getName()))) {
+ // add generated columns to schema.
+ transactionState.addColumn(partitionField);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void syncMetadata(TableSyncMetadata metadata) {
+ transactionState.setMetadata(metadata);
+ }
+
+ @Override
+ public void syncFilesForSnapshot(List<PartitionFileGroup>
partitionedDataFiles) {
+ Table table = Table.forPath(engine, basePath);
+ // Pass cached snapshot to avoid reloading it
+ transactionState.setActions(
+ dataKernelFileUpdatesExtractor.applySnapshot(
+ table,
+ partitionedDataFiles,
+ transactionState.getLatestSchemaInternal(),
+ transactionState.getCachedSnapshot()));
+ }
+
+ @Override
+ public void syncFilesForDiff(InternalFilesDiff internalFilesDiff) {
+ // Use cached schema directly (already extracted during TransactionState
initialization)
+ // For new tables, latestSchema is null and applyDiff will handle it
appropriately
+ transactionState.setActions(
+ dataKernelFileUpdatesExtractor.applyDiff(
+ internalFilesDiff,
+ transactionState.getLatestSchemaInternal(),
+ basePath,
+ transactionState.getLatestSchema()));
+ }
+
+ @Override
+ public void completeSync() {
+ transactionState.commitTransaction();
+ transactionState = null;
+ }
+
+ @Override
+ public Optional<TableSyncMetadata> getTableMetadata() {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getLatestSnapshot(engine);
+
+ // WORKAROUND: Cast to SnapshotImpl (internal class) to access metadata
configuration.
+ // Delta Kernel 4.0.0 does not provide a public API to access table
metadata/configuration.
+ // This cast is brittle and may break on Kernel version upgrades.
+ Metadata metadata = ((SnapshotImpl) snapshot).getMetadata();
+
+ // Get configuration from metadata
+ Map<String, String> configuration = metadata.getConfiguration();
+ String metadataJson = configuration.get(TableSyncMetadata.XTABLE_METADATA);
+
+ return TableSyncMetadata.fromJson(metadataJson);
+ }
+
+ @Override
+ public String getTableFormat() {
+ return TableFormat.DELTA;
+ }
+
+ @Override
+ public Optional<String> getTargetCommitIdentifier(String sourceIdentifier) {
+ // Delta Kernel 4.0.0 does not support commit tags in commitInfo, which
are required for
+ // source-to-target commit identifier mapping. This limitation is
documented in:
+ // https://github.com/delta-io/delta/issues/6167
+ // XTable tracking issue:
https://github.com/apache/incubator-xtable/issues/819
+ //
+ // Unlike DeltaConversionTarget (which uses Delta Standalone with commit
tag support),
+ // DeltaKernelConversionTarget cannot retrieve commit tags from Delta
Kernel's API.
+ // Rather than silently scanning all commits (O(n) performance cost) and
always returning
+ // empty, we explicitly throw an exception to indicate this feature is
unsupported.
+ //
+ // When Delta Kernel adds commit tag support, this method can be
reimplemented to:
+ // 1. Scan commit history using tableImpl.getChanges(engine, 0,
currentVersion, actionSet)
+ // 2. Extract tags from CommitInfo.tags MapValue
+ // 3. Parse XTABLE_METADATA from tags and match sourceIdentifier
+ throw new NotSupportedException(
+ "Source-to-target commit identifier mapping is not supported in
DeltaKernelConversionTarget. "
+ + "Delta Kernel 4.0.0 does not support commit tags in commitInfo. "
+ + "See: https://github.com/delta-io/delta/issues/6167");
+ }
+
+ private class TransactionState {
+ private final Engine engine;
+ private final long retentionInHours;
+ private final List<String> partitionColumns;
+ @Getter private StructType latestSchema;
+ @Getter private InternalSchema latestSchemaInternal;
+ private TableSyncMetadata metadata;
+ private List<RowBackedAction> actions;
+
+ // Cache the snapshot to avoid multiple loads during a single sync cycle
+ // This snapshot is used for: schema extraction, table existence check,
and file diff operations
+ private Snapshot cachedSnapshot;
+ private boolean tableExists;
+
+ private TransactionState(Engine engine, long retentionInHours) {
+ this.engine = engine;
+ this.partitionColumns = new ArrayList<>();
+ this.retentionInHours = retentionInHours;
+
+ try {
+ Table table = Table.forPath(engine, basePath);
+ this.cachedSnapshot = table.getLatestSnapshot(engine);
+ this.latestSchema = cachedSnapshot.getSchema();
+ this.tableExists = true;
+ } catch (TableNotFoundException e) {
+ // Expected: table doesn't exist yet on first sync
+ this.latestSchema = null;
+ this.cachedSnapshot = null;
+ this.tableExists = false;
+ }
+ // Let other exceptions propagate (network issues, permissions,
corrupted metadata, etc.)
+ }
+
+ /**
+ * Adds a partition column name to the list. Package-private to allow
access from outer class.
+ */
+ void addPartitionColumn(String columnName) {
+ partitionColumns.add(columnName);
+ }
+
+ /**
+ * Gets the cached snapshot. Returns null if no snapshot was cached (new
table). Package-private
+ * to allow access from outer class.
+ */
+ Snapshot getCachedSnapshot() {
+ return cachedSnapshot;
+ }
+
+ void setMetadata(TableSyncMetadata metadata) {
+ this.metadata = metadata;
+ }
+
+ /**
+ * Sets the actions to be committed. Converts from Scala Seq to Java List
for internal storage.
+ */
+ void setActions(Seq<RowBackedAction> scalaActions) {
+ this.actions = JavaConverters.seqAsJavaList(scalaActions);
+ }
+
+ private void addColumn(StructField field) {
+ latestSchema = latestSchema.add(field);
+ latestSchemaInternal = schemaExtractor.toInternalSchema(latestSchema);
+ }
+
+ private void setLatestSchema(InternalSchema schema) {
+ this.latestSchemaInternal = schema;
+ this.latestSchema = schemaExtractor.fromInternalSchema(schema);
+ }
+
+ private void commitTransaction() {
+ // Use cached table existence check instead of loading snapshot again
+ Operation operation = tableExists ? Operation.WRITE :
Operation.CREATE_TABLE;
+
+ Table table = Table.forPath(engine, basePath);
+ TransactionBuilder txnBuilder =
+ table.createTransactionBuilder(engine, "XTable Delta Sync",
operation);
+
+ // LIMITATION: Schema evolution for existing tables is NOT supported in
Delta Kernel 4.0.0.
+ // The withSchema() method only works during CREATE_TABLE operations.
For existing tables:
+ // - AddFile/RemoveFile actions are created using the old schema from
existing snapshot
+ // - If source schema has evolved (columns added/removed/type changed),
the Delta table
+ // will have mismatched metadata and data, causing query failures or
incorrect results
+ // This is a known Delta Kernel limitation:
https://github.com/delta-io/delta/issues/4305
+ if (!tableExists) {
+ txnBuilder = txnBuilder.withSchema(engine, latestSchema);
+
+ if (!partitionColumns.isEmpty()) {
+ txnBuilder = txnBuilder.withPartitionColumns(engine,
partitionColumns);
+ }
+ }
+
+ Map<String, String> tableProperties = getConfigurationsForDeltaSync();
+ txnBuilder = txnBuilder.withTableProperties(engine, tableProperties);
+
+ Transaction txn = txnBuilder.build(engine);
+ List<Row> allActionRows = new ArrayList<>();
+
+ // Iterate through actions (Java List) and convert to Row format
+ for (RowBackedAction action : actions) {
+
+ if (action instanceof AddFile) {
+ AddFile addFile = (AddFile) action;
+ Row wrappedRow =
SingleAction.createAddFileSingleAction(addFile.toRow());
+ allActionRows.add(wrappedRow);
+ } else if (action instanceof RemoveFile) {
+ RemoveFile removeFile = (RemoveFile) action;
+ Row wrappedRow =
SingleAction.createRemoveFileSingleAction(removeFile.toRow());
+ allActionRows.add(wrappedRow);
+ }
+ }
+
+ CloseableIterator<Row> allActionsIterator =
+ new CloseableIterator<Row>() {
+ private int currentIndex = 0;
+
+ @Override
+ public boolean hasNext() {
+ return currentIndex < allActionRows.size();
+ }
+
+ @Override
+ public Row next() {
+ return allActionRows.get(currentIndex++);
+ }
+
+ @Override
+ public void close() {}
+ };
+
+ CloseableIterable<Row> dataActions =
inMemoryIterable(allActionsIterator);
+
+ try {
+ TransactionCommitResult result = txn.commit(engine, dataActions);
+
+ // Execute PostCommitHooks to create checkpoints and _last_checkpoint
metadata file
+ List<PostCommitHook> hooks = result.getPostCommitHooks();
+ if (hooks != null && !hooks.isEmpty()) {
+ for (PostCommitHook hook : hooks) {
+ try {
+ hook.threadSafeInvoke(engine);
+ } catch (Exception hookEx) {
+ // Post-commit hooks are optimizations; log but don't fail the
transaction
+ log.warn("Post-commit hook failed but transaction succeeded",
hookEx);
Review Comment:
The post-commit-hook warning is missing context — include the table
`basePath` so on-call can identify which table's checkpointing failed:
`log.warn("Post-commit hook failed for table {}", basePath, hookEx)`.
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileUpdatesExtractor.java:
##########
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import lombok.Builder;
+
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.MapValue;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.expressions.Column;
+import io.delta.kernel.expressions.Literal;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.util.VectorUtils;
+import io.delta.kernel.statistics.DataFileStatistics;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.exception.ReadException;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.FilesDiff;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalFile;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.paths.PathUtils;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+@Builder
+public class DeltaKernelDataFileUpdatesExtractor {
+ @Builder.Default
+ private final DeltaKernelStatsExtractor deltaStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor deltaKernelPartitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelDataFileExtractor dataFileExtractor =
+ DeltaKernelDataFileExtractor.builder().build();
+
+ private final Engine engine;
+ private final String basePath;
+ private final boolean includeColumnStats;
+
+ /**
+ * Applies snapshot changes, loading the snapshot fresh.
+ *
+ * @param table the Delta table
+ * @param partitionedDataFiles the new data files to sync
+ * @param tableSchema the internal schema
+ * @return sequence of Delta actions (AddFile/RemoveFile)
+ */
+ public Seq<RowBackedAction> applySnapshot(
+ Table table, List<PartitionFileGroup> partitionedDataFiles,
InternalSchema tableSchema) {
+ return applySnapshot(table, partitionedDataFiles, tableSchema, null);
+ }
+
+ /**
+ * Applies snapshot changes with an optional cached snapshot to avoid
redundant loads.
+ *
+ * @param table the Delta table
+ * @param partitionedDataFiles the new data files to sync
+ * @param tableSchema the internal schema
+ * @param cachedSnapshot optional pre-loaded snapshot (null to load fresh)
+ * @return sequence of Delta actions (AddFile/RemoveFile)
+ */
+ public Seq<RowBackedAction> applySnapshot(
+ Table table,
+ List<PartitionFileGroup> partitionedDataFiles,
+ InternalSchema tableSchema,
+ Snapshot cachedSnapshot) {
+
+ // all files in the current delta snapshot are potential candidates for
remove actions, i.e. if
+ // the file is not present in the new snapshot (addedFiles) then the file
is considered removed
+ Map<String, RowBackedAction> previousFiles = new HashMap<>();
+ StructType physicalSchema;
+
+ // Use cached snapshot if provided, otherwise load it
+ boolean tableExists = cachedSnapshot != null || checkTableExists(table);
+
+ if (tableExists) {
+ Snapshot snapshot = cachedSnapshot != null ? cachedSnapshot :
table.getLatestSnapshot(engine);
+
+ // Reuse DeltaKernelDataFileExtractor to iterate through existing files
+ // This avoids duplicating the scan logic for reading Delta files
+ try (DataFileIterator fileIterator =
+ dataFileExtractor.iterator(snapshot, table, engine, tableSchema)) {
+
+ while (fileIterator.hasNext()) {
+ InternalDataFile internalFile = fileIterator.next();
+
+ // Convert InternalDataFile back to AddFile to create RemoveFile
action
+ AddFile addFile =
+ createAddFileAction(internalFile, tableSchema, basePath,
snapshot.getSchema());
+ RemoveFile removeFile =
+ new RemoveFile(addFile.toRemoveFileRow(false,
Optional.of(snapshot.getVersion())));
+ // Use optimized path construction with Engine (reuses existing
engine, no new
+ // Configuration)
+ String fullPath =
+
DeltaKernelActionsConverter.getFullPathToFile(removeFile.getPath(), engine,
table);
+ previousFiles.put(fullPath, removeFile);
+ }
+ } catch (Exception e) {
+ throw new ReadException("Failed to scan existing Delta files", e);
+ }
+
+ physicalSchema = snapshot.getSchema();
+
+ } else {
+
+ // Table doesn't exist yet - no previous files to remove
+ // Convert InternalSchema to StructType for physical schema
+ DeltaKernelSchemaExtractor schemaExtractor =
DeltaKernelSchemaExtractor.getInstance();
+ physicalSchema = schemaExtractor.fromInternalSchema(tableSchema);
+ }
+
+ FilesDiff<InternalFile, RowBackedAction> diff =
+ InternalFilesDiff.findNewAndRemovedFiles(partitionedDataFiles,
previousFiles);
+
+ return applyDiff(
+ diff.getFilesAdded(), diff.getFilesRemoved(), tableSchema, basePath,
physicalSchema);
+ }
+
+ private boolean checkTableExists(Table table) {
+ return DeltaKernelUtils.tableExists(engine, table.getPath(engine));
+ }
+
+ public Seq<RowBackedAction> applyDiff(
+ InternalFilesDiff internalFilesDiff,
+ InternalSchema tableSchema,
+ String tableBasePath,
+ StructType physicalSchema) {
+ List<RowBackedAction> removeActions =
+ internalFilesDiff.dataFilesRemoved().stream()
+ .map(dFile -> createAddFileAction(dFile, tableSchema,
tableBasePath, physicalSchema))
+ .map(addFile -> new RemoveFile(addFile.toRemoveFileRow(false,
Optional.empty())))
+
.collect(CustomCollectors.toList(internalFilesDiff.dataFilesRemoved().size()));
+ return applyDiff(
+ internalFilesDiff.dataFilesAdded(),
+ removeActions,
+ tableSchema,
+ tableBasePath,
+ physicalSchema);
+ }
+
+ private Seq<RowBackedAction> applyDiff(
+ Set<? extends InternalFile> filesAdded,
+ Collection<RowBackedAction> removeFileActions,
+ InternalSchema tableSchema,
+ String tableBasePath,
+ StructType physicalSchema) {
+ Stream<RowBackedAction> addActions =
+ filesAdded.stream()
+ .filter(InternalDataFile.class::isInstance)
+ .map(file -> (InternalDataFile) file)
+ .map(dFile -> createAddFileAction(dFile, tableSchema,
tableBasePath, physicalSchema));
+ int totalActions = filesAdded.size() + removeFileActions.size();
+ List<RowBackedAction> allActions =
+ Stream.concat(addActions, removeFileActions.stream())
+ .collect(CustomCollectors.toList(totalActions));
+ return JavaConverters.asScalaBuffer(allActions).toSeq();
+ }
+
+ private AddFile createAddFileAction(
+ InternalDataFile dataFile,
+ InternalSchema tableSchema,
+ String tableBasePath,
+ StructType physicalSchema) {
+ // Convert partition values from Map<String, String> to MapValue
+ Map<String, String> partitionValuesMap =
+ deltaKernelPartitionExtractor.partitionValueSerialization(dataFile);
+ MapValue partitionValues = convertToMapValue(partitionValuesMap);
+
+ // Generate column stats if enabled
+ Optional<DataFileStatistics> stats = Optional.empty();
+ if (includeColumnStats) {
+ stats =
+ Optional.of(
+ convertToDataFileStatistics(dataFile.getRecordCount(),
dataFile.getColumnStats()));
+ }
+
+ Row addFileRow =
+ AddFile.createAddFileRow(
+ physicalSchema,
+ // Delta Lake supports relative and absolute paths in theory but
relative paths seem
+ // more commonly supported by query engines in our testing
+ PathUtils.getRelativePath(dataFile.getPhysicalPath(),
tableBasePath),
+ partitionValues,
+ dataFile.getFileSizeBytes(),
+ dataFile.getLastModified(),
+ true, // dataChange
+ Optional.empty(), // deletionVector
+ Optional.empty(), // tags
+ Optional.empty(), // baseRowId
+ Optional.empty(), // defaultRowCommitVersion
+ stats // stats - converted from InternalDataFile column stats
+ );
+
+ // Wrap the Row back into an AddFile object so we can use its methods
+ return new AddFile(addFileRow);
+ }
+
+ /**
+ * Converts XTable's internal column statistics to Delta Kernel's
DataFileStatistics format.
+ *
+ * @param recordCount the number of records in the file
+ * @param columnStats the list of column statistics from InternalDataFile
+ * @return DataFileStatistics object for Delta Kernel
+ */
+ private DataFileStatistics convertToDataFileStatistics(
+ long recordCount, List<ColumnStat> columnStats) {
+ if (columnStats == null || columnStats.isEmpty()) {
+ return new DataFileStatistics(recordCount, new HashMap<>(), new
HashMap<>(), new HashMap<>());
+ }
+
+ Map<Column, Literal> minValues = new HashMap<>();
+ Map<Column, Literal> maxValues = new HashMap<>();
+ Map<Column, Long> nullCounts = new HashMap<>();
+
+ for (ColumnStat columnStat : columnStats) {
+ InternalField field = columnStat.getField();
+ InternalType dataType = field.getSchema().getDataType();
+
+ // Only process supported types for statistics
+ if (!isSupportedStatsType(dataType)) {
+ continue;
+ }
+
+ // Create Column reference from field path
+ Column column = new Column(field.getPathParts());
+
+ // Extract min/max from range
+ Range range = columnStat.getRange();
+ if (range != null) {
+ Object minValue = range.getMinValue();
+ Object maxValue = range.getMaxValue();
+
+ if (minValue != null) {
+ Literal minLiteral = convertToLiteral(minValue, field.getSchema());
+ if (minLiteral != null) {
+ minValues.put(column, minLiteral);
+ }
+ }
+
+ if (maxValue != null) {
+ Literal maxLiteral = convertToLiteral(maxValue, field.getSchema());
+ if (maxLiteral != null) {
+ maxValues.put(column, maxLiteral);
+ }
+ }
+ }
+
+ // Add null count
+ nullCounts.put(column, columnStat.getNumNulls());
+ }
+
+ return new DataFileStatistics(recordCount, minValues, maxValues,
nullCounts);
+ }
+
+ /** Checks if a data type is supported for statistics. */
+ private boolean isSupportedStatsType(InternalType type) {
+ return type == InternalType.BOOLEAN
+ || type == InternalType.DATE
+ || type == InternalType.DECIMAL
+ || type == InternalType.DOUBLE
+ || type == InternalType.INT
+ || type == InternalType.LONG
+ || type == InternalType.FLOAT
+ || type == InternalType.STRING
+ || type == InternalType.TIMESTAMP
+ || type == InternalType.TIMESTAMP_NTZ;
+ }
+
+ /** Converts an XTable value to a Delta Kernel Literal based on the field
schema. */
+ private Literal convertToLiteral(Object value, InternalSchema fieldSchema) {
+ InternalType dataType = fieldSchema.getDataType();
+
+ switch (dataType) {
+ case BOOLEAN:
+ return Literal.ofBoolean((Boolean) value);
+ case INT:
+ return Literal.ofInt(((Number) value).intValue());
+ case LONG:
+ return Literal.ofLong(((Number) value).longValue());
+ case FLOAT:
+ return Literal.ofFloat(((Number) value).floatValue());
+ case DOUBLE:
+ return Literal.ofDouble(((Number) value).doubleValue());
+ case STRING:
+ return Literal.ofString((String) value);
+ case DATE:
+ // XTable stores dates as days since epoch (int)
+ return Literal.ofDate(((Number) value).intValue());
+ case TIMESTAMP:
+ // XTable stores timestamps as microseconds since epoch
+ return Literal.ofTimestamp(((Number) value).longValue());
+ case TIMESTAMP_NTZ:
+ // XTable stores timestamp_ntz as microseconds since epoch
+ return Literal.ofTimestampNtz(((Number) value).longValue());
+ case DECIMAL:
+ // Extract precision and scale from schema metadata
+ Map<InternalSchema.MetadataKey, Object> metadata =
fieldSchema.getMetadata();
+ int precision = 10; // default precision
Review Comment:
Silent corruption risk: defaulting `precision=10, scale=0` when metadata is
missing produces wrong stats — e.g., a `DECIMAL(38,18)` value would be written
with scale=0, breaking min/max comparisons downstream. Throw
`IllegalStateException` (or use `Preconditions.checkArgument`) when
precision/scale aren't present for a DECIMAL field instead of falling back to
defaults.
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelActionsConverter.java:
##########
@@ -104,10 +176,50 @@ public FileFormat convertToFileFormat(String provider) {
String.format("delta file format %s is not recognized", provider));
}
+ /**
+ * Constructs the full path to a file, handling both relative and absolute
paths.
+ *
+ * <p><strong>DEPRECATED:</strong> This method creates a new Configuration
and Engine on every
+ * call, which is extremely inefficient. Use {@link
#getFullPathToFile(String, Engine, Table)} or
+ * {@link #getFullPathToFile(String, String)} instead.
+ *
+ * @param dataFilePath the data file path (relative or absolute)
+ * @param table the Delta table
+ * @return the full absolute path to the file
+ * @deprecated Use {@link #getFullPathToFile(String, Engine, Table)} to
avoid creating new
+ * Engine/Configuration on every call, or {@link
#getFullPathToFile(String, String)} if you
+ * already have the base path
+ */
+ @Deprecated
static String getFullPathToFile(String dataFilePath, Table table) {
+ // WARNING: This creates new Configuration + Engine per call - severe
performance issue!
+ // Kept for backwards compatibility but should not be used in hot paths
Configuration hadoopConf = new Configuration();
Engine myEngine = DefaultEngine.create(hadoopConf);
- String tableBasePath = table.getPath(myEngine);
+ return getFullPathToFile(dataFilePath, myEngine, table);
+ }
+
+ /**
+ * Constructs the full path to a file using a provided Engine (efficient).
+ *
+ * @param dataFilePath the data file path (relative or absolute)
+ * @param engine the Delta Kernel engine to use for path resolution
+ * @param table the Delta table
+ * @return the full absolute path to the file
+ */
+ static String getFullPathToFile(String dataFilePath, Engine engine, Table
table) {
+ String tableBasePath = table.getPath(engine);
+ return getFullPathToFile(dataFilePath, tableBasePath);
+ }
+
+ /**
+ * Constructs the full path to a file using a provided base path (most
efficient).
+ *
+ * @param dataFilePath the data file path (relative or absolute)
+ * @param tableBasePath the table base path
+ * @return the full absolute path to the file
+ */
+ static String getFullPathToFile(String dataFilePath, String tableBasePath) {
Review Comment:
`dataFilePath.startsWith(tableBasePath)` is a fragile prefix check. With
`tableBasePath = "/foo"` and `dataFilePath = "/foobar/x.parquet"`, it returns
true and the path won't be normalized — but the file isn't actually under the
base. Either compare with a trailing separator (`tableBasePath +
Path.SEPARATOR`) or use `URI`/`Path` resolution.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]