This is an automated email from the ASF dual-hosted git repository.
vinishjail97 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-xtable.git
The following commit(s) were added to refs/heads/main by this push:
new 244003d2 fix(delta): null partition value for composite
generated-column partitions (#828)
244003d2 is described below
commit 244003d2d61f9371713c711236bb0cb0f1eda79a
Author: Aleksandr Andreev <[email protected]>
AuthorDate: Mon Jun 22 22:52:24 2026 +0300
fix(delta): null partition value for composite generated-column partitions
(#828)
* Fix composite partition value corruption when a component is missing
DeltaPartitionExtractor#getSerializedPartitionValue joined the values of
a composite (generated column) partition with Collectors.joining("-").
When one of the component values was absent from the partition values
map, the missing value was rendered as the literal string "null",
producing a corrupted partition value such as "2013-null-20" that was
then fed to the date parser.
Return null when any component value is missing so the partition value
resolves to null, consistent with the single-field branch that uses
getOrDefault(name, null). Added a regression test covering a composite
generated-column partition with a missing component.
* Apply composite partition null-component fix to Delta Kernel extractor
DeltaKernelPartitionExtractor#getSerializedPartitionValue contained the
same composite (generated column) partition handling as the Spark-based
DeltaPartitionExtractor: joining component values with
Collectors.joining("-") rendered a missing component as the literal
string "null", corrupting the partition value.
Apply the same fix here - return null when any component value is
missing - and add a regression test mirroring the one added for
DeltaPartitionExtractor.
* Strengthen Delta composite partition tests with null-source scenario
The composite (generated column) partition columns are all derived from
a single source column, so the realistic trigger for a missing component
is a null source value, which makes every derived partition column null.
- Reframe the DeltaPartitionExtractor and DeltaKernelPartitionExtractor
unit tests around that null-source case (all components null) instead
of an artificial single-missing-component map.
- Add an integration test (ITDeltaConversionSource) that creates a Delta
table partitioned by year/month/day generated columns derived from a
nullable timestamp, inserts a row with a null timestamp, and verifies
the snapshot resolves the partition value to null. Without the fix this
reproduces the failure (ParseException on "null-null-null").
* Address review feedback
- Create the gen-col partitioned test table via the DeltaTable builder API
instead of CREATE TABLE SQL (Spark 3.4 rejects generated partition columns
via SQL), and use a named-column INSERT so generated columns are computed.
- Drop misleading scalaMap alias in the kernel test.
- Tighten comments.
---------
Co-authored-by: Aleksandr Andreev <persistentengineer.com>
Co-authored-by: Vinish Reddy <[email protected]>
---
.../xtable/delta/DeltaPartitionExtractor.java | 8 ++-
.../kernel/DeltaKernelPartitionExtractor.java | 8 ++-
.../xtable/delta/ITDeltaConversionSource.java | 69 ++++++++++++++++++++++
.../xtable/delta/TestDeltaPartitionExtractor.java | 60 +++++++++++++++++++
.../kernel/TestDeltaKernelPartitionExtractor.java | 58 ++++++++++++++++++
5 files changed, 197 insertions(+), 6 deletions(-)
diff --git
a/xtable-core/src/main/java/org/apache/xtable/delta/DeltaPartitionExtractor.java
b/xtable-core/src/main/java/org/apache/xtable/delta/DeltaPartitionExtractor.java
index 7d9db06e..326b03c0 100644
---
a/xtable-core/src/main/java/org/apache/xtable/delta/DeltaPartitionExtractor.java
+++
b/xtable-core/src/main/java/org/apache/xtable/delta/DeltaPartitionExtractor.java
@@ -326,9 +326,11 @@ public class DeltaPartitionExtractor {
if (partitionFieldNames.size() == 1) {
return values.getOrDefault(partitionFieldNames.get(0), null);
}
- return partitionFieldNames.stream()
- .map(name -> values.get(name))
- .collect(Collectors.joining("-"));
+ // Composite partition: any null component yields a null value, not joined
"null"s.
+ if (partitionFieldNames.stream().anyMatch(name -> values.get(name) ==
null)) {
+ return null;
+ }
+ return
partitionFieldNames.stream().map(values::get).collect(Collectors.joining("-"));
}
private String getGeneratedColumnName(InternalPartitionField
internalPartitionField) {
diff --git
a/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelPartitionExtractor.java
b/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelPartitionExtractor.java
index 95ffc9d4..4a236867 100644
---
a/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelPartitionExtractor.java
+++
b/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelPartitionExtractor.java
@@ -319,9 +319,11 @@ public class DeltaKernelPartitionExtractor {
if (partitionFieldNames.size() == 1) {
return values.getOrDefault(partitionFieldNames.get(0), null);
}
- return partitionFieldNames.stream()
- .map(name -> values.get(name))
- .collect(Collectors.joining("-"));
+ // Composite partition: any null component yields a null value, not joined
"null"s.
+ if (partitionFieldNames.stream().anyMatch(name -> values.get(name) ==
null)) {
+ return null;
+ }
+ return
partitionFieldNames.stream().map(values::get).collect(Collectors.joining("-"));
}
private String getGeneratedColumnName(InternalPartitionField
internalPartitionField) {
diff --git
a/xtable-core/src/test/java/org/apache/xtable/delta/ITDeltaConversionSource.java
b/xtable-core/src/test/java/org/apache/xtable/delta/ITDeltaConversionSource.java
index 80ba46a3..326a98d8 100644
---
a/xtable-core/src/test/java/org/apache/xtable/delta/ITDeltaConversionSource.java
+++
b/xtable-core/src/test/java/org/apache/xtable/delta/ITDeltaConversionSource.java
@@ -18,6 +18,9 @@
package org.apache.xtable.delta;
+import static org.apache.spark.sql.types.DataTypes.IntegerType;
+import static org.apache.spark.sql.types.DataTypes.LongType;
+import static org.apache.spark.sql.types.DataTypes.TimestampType;
import static org.apache.xtable.testutil.ITTestUtils.validateTable;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -52,6 +55,8 @@ import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
+import io.delta.tables.DeltaTable;
+
import org.apache.xtable.GenericTable;
import org.apache.xtable.TestSparkDeltaTable;
import org.apache.xtable.ValidationTestHelper;
@@ -368,6 +373,70 @@ public class ITDeltaConversionSource {
InternalSnapshot snapshot = conversionSource.getCurrentSnapshot();
}
+ @Test
+ void getCurrentSnapshotGenColPartitionedWithNullSourceTest() {
+ // year/month/day partition columns generated from one source column; a
null source produces a
+ // null component for each, which must resolve to a null partition value,
not "null-null-null".
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ // Builder API, not CREATE TABLE SQL: Spark 3.4 rejects generated
partition columns via SQL.
+ DeltaTable.createIfNotExists(sparkSession)
+ .tableName(tableName)
+ .location(basePath.toString())
+ .addColumn("id", LongType)
+ .addColumn("event_time", TimestampType)
+ .addColumn(
+ DeltaTable.columnBuilder("year_col")
+ .dataType(IntegerType)
+ .generatedAlwaysAs("YEAR(event_time)")
+ .build())
+ .addColumn(
+ DeltaTable.columnBuilder("month_col")
+ .dataType(IntegerType)
+ .generatedAlwaysAs("MONTH(event_time)")
+ .build())
+ .addColumn(
+ DeltaTable.columnBuilder("day_col")
+ .dataType(IntegerType)
+ .generatedAlwaysAs("DAY(event_time)")
+ .build())
+ .partitionedBy("year_col", "month_col", "day_col")
+ .execute();
+ // Second row has a null timestamp, so every generated partition column is
null.
+ sparkSession.sql(
+ "INSERT INTO TABLE `"
+ + tableName
+ + "` (id, event_time) VALUES (1, CAST('2013-08-20 00:00:00' AS
TIMESTAMP)),"
+ + " (2, CAST(NULL AS TIMESTAMP))");
+
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+
+ InternalSnapshot snapshot = conversionSource.getCurrentSnapshot();
+
+ List<InternalDataFile> dataFiles =
+ snapshot.getPartitionedDataFiles().stream()
+ .flatMap(group -> group.getDataFiles().stream())
+ .collect(Collectors.toList());
+ assertEquals(2, dataFiles.size());
+
+ // Exactly one file is in the null-source partition, with a null partition
value.
+ long nullPartitionFiles =
+ dataFiles.stream()
+ .filter(
+ file ->
+ file.getPartitionValues().stream()
+ .anyMatch(pv -> pv.getRange().getMaxValue() == null))
+ .count();
+ assertEquals(1, nullPartitionFiles);
+ }
+
@ParameterizedTest
@MethodSource("testWithPartitionToggle")
public void testInsertsUpsertsAndDeletes(boolean isPartitioned) {
diff --git
a/xtable-core/src/test/java/org/apache/xtable/delta/TestDeltaPartitionExtractor.java
b/xtable-core/src/test/java/org/apache/xtable/delta/TestDeltaPartitionExtractor.java
index 5b767096..a405de08 100644
---
a/xtable-core/src/test/java/org/apache/xtable/delta/TestDeltaPartitionExtractor.java
+++
b/xtable-core/src/test/java/org/apache/xtable/delta/TestDeltaPartitionExtractor.java
@@ -496,6 +496,66 @@ public class TestDeltaPartitionExtractor {
assertEquals(expectedPartitionValues, partitionValues);
}
+ @Test
+ public void testGeneratedPartitionValueExtractionWithNullSource() {
+ // Composite partition whose components (year/month/day) are all null must
resolve to a null
+ // partition value, not the literal "null-null-null".
+ Map<String, String> partitionValuesMap =
+ new HashMap<String, String>() {
+ {
+ put("partition_column1", "partition_value1");
+ put("year_partition_column", null);
+ put("month_partition_column", null);
+ put("day_partition_column", null);
+ }
+ };
+ scala.collection.mutable.Map<String, String> scalaMap =
+ convertJavaMapToScalaMap(partitionValuesMap);
+ InternalPartitionField internalPartitionField1 =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("partition_column1")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build();
+ InternalPartitionField internalPartitionField2 =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("some_date_column")
+ .schema(
+ InternalSchema.builder()
+ .name("timestamp")
+ .dataType(InternalType.TIMESTAMP)
+ .build())
+ .build())
+ .partitionFieldNames(
+ Arrays.asList(
+ "year_partition_column", "month_partition_column",
"day_partition_column"))
+ .transformType(PartitionTransformType.DAY)
+ .build();
+ List<PartitionValue> expectedPartitionValues =
+ Arrays.asList(
+ PartitionValue.builder()
+ .partitionField(internalPartitionField1)
+ .range(Range.scalar("partition_value1"))
+ .build(),
+ PartitionValue.builder()
+ .partitionField(internalPartitionField2)
+ .range(Range.scalar(null))
+ .build());
+ List<PartitionValue> partitionValues =
+ deltaPartitionExtractor.partitionValueExtraction(
+ scalaMap, Arrays.asList(internalPartitionField1,
internalPartitionField2));
+ assertEquals(expectedPartitionValues, partitionValues);
+ }
+
@Test
void convertBucketPartition() {
InternalPartitionField internalPartitionField =
diff --git
a/xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelPartitionExtractor.java
b/xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelPartitionExtractor.java
index 536bd116..bfdbaf8f 100644
---
a/xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelPartitionExtractor.java
+++
b/xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelPartitionExtractor.java
@@ -513,6 +513,64 @@ public class TestDeltaKernelPartitionExtractor {
assertEquals(expectedPartitionValues, partitionValues);
}
+ @Test
+ public void testGeneratedPartitionValueExtractionWithNullSource() {
+ // Composite partition whose components (year/month/day) are all null must
resolve to a null
+ // partition value, not the literal "null-null-null".
+ Map<String, String> partitionValuesMap =
+ new HashMap<String, String>() {
+ {
+ put("partition_column1", "partition_value1");
+ put("year_partition_column", null);
+ put("month_partition_column", null);
+ put("day_partition_column", null);
+ }
+ };
+ InternalPartitionField internalPartitionField1 =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("partition_column1")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build();
+ InternalPartitionField internalPartitionField2 =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("some_date_column")
+ .schema(
+ InternalSchema.builder()
+ .name("timestamp")
+ .dataType(InternalType.TIMESTAMP)
+ .build())
+ .build())
+ .partitionFieldNames(
+ Arrays.asList(
+ "year_partition_column", "month_partition_column",
"day_partition_column"))
+ .transformType(PartitionTransformType.DAY)
+ .build();
+ List<PartitionValue> expectedPartitionValues =
+ Arrays.asList(
+ PartitionValue.builder()
+ .partitionField(internalPartitionField1)
+ .range(Range.scalar("partition_value1"))
+ .build(),
+ PartitionValue.builder()
+ .partitionField(internalPartitionField2)
+ .range(Range.scalar(null))
+ .build());
+ List<PartitionValue> partitionValues =
+ deltaKernelPartitionExtractor.partitionValueExtraction(
+ partitionValuesMap, Arrays.asList(internalPartitionField1,
internalPartitionField2));
+ assertEquals(expectedPartitionValues, partitionValues);
+ }
+
@Test
void convertBucketPartition() {
InternalPartitionField internalPartitionField =
