This is an automated email from the ASF dual-hosted git repository.
vinishjail97 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-xtable.git
The following commit(s) were added to refs/heads/main by this push:
new a4e729d6 Fix map key path handling in Delta schema extractors (#826)
a4e729d6 is described below
commit a4e729d61b9bebb048e415a047cb330c71e2d43d
Author: Rishi Reddy Bokka <[email protected]>
AuthorDate: Fri Jun 5 11:04:15 2026 -0700
Fix map key path handling in Delta schema extractors (#826)
* fix(delta): qualify nested map-key fields with MAP_KEY_FIELD_NAME (Spark
+ Kernel)
* fix(delta-kernel): null-guard originalMetadata in BinaryType branch
* ci: retrigger build
---
.../apache/xtable/delta/DeltaSchemaExtractor.java | 2 +-
.../xtable/kernel/DeltaKernelSchemaExtractor.java | 7 +-
.../xtable/delta/TestDeltaSchemaExtractor.java | 92 +++++++++++
.../kernel/TestDeltaKernelSchemaExtractor.java | 168 +++++++++++++++++++++
4 files changed, 266 insertions(+), 3 deletions(-)
diff --git
a/xtable-core/src/main/java/org/apache/xtable/delta/DeltaSchemaExtractor.java
b/xtable-core/src/main/java/org/apache/xtable/delta/DeltaSchemaExtractor.java
index d6e59a97..f72d5a6a 100644
---
a/xtable-core/src/main/java/org/apache/xtable/delta/DeltaSchemaExtractor.java
+++
b/xtable-core/src/main/java/org/apache/xtable/delta/DeltaSchemaExtractor.java
@@ -196,7 +196,7 @@ public class DeltaSchemaExtractor {
toInternalSchema(
mapType.keyType(),
SchemaUtils.getFullyQualifiedPath(
- parentPath, InternalField.Constants.MAP_VALUE_FIELD_NAME),
+ parentPath, InternalField.Constants.MAP_KEY_FIELD_NAME),
false,
null,
null);
diff --git
a/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java
b/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java
index 8055ee4f..464ec2bb 100644
---
a/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java
+++
b/xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java
@@ -96,7 +96,8 @@ public class DeltaKernelSchemaExtractor {
type = InternalType.DOUBLE;
trimmedTypeName = "double";
} else if (dataType instanceof BinaryType) {
- if (originalMetadata.contains(InternalSchema.XTABLE_LOGICAL_TYPE)
+ if (originalMetadata != null
+ && originalMetadata.contains(InternalSchema.XTABLE_LOGICAL_TYPE)
&&
"uuid".equals(originalMetadata.getString(InternalSchema.XTABLE_LOGICAL_TYPE))) {
type = InternalType.UUID;
trimmedTypeName = "binary";
@@ -193,7 +194,7 @@ public class DeltaKernelSchemaExtractor {
toInternalSchema(
mapType.getKeyType(),
SchemaUtils.getFullyQualifiedPath(
- parentPath, InternalField.Constants.MAP_VALUE_FIELD_NAME),
+ parentPath, InternalField.Constants.MAP_KEY_FIELD_NAME),
false,
null,
null);
@@ -220,6 +221,8 @@ public class DeltaKernelSchemaExtractor {
type = InternalType.MAP;
fields = Arrays.asList(keyField, valueField);
trimmedTypeName = "map";
+ } else {
+ throw new NotSupportedException("Unsupported type: " + dataType);
}
return InternalSchema.builder()
.name(trimmedTypeName)
diff --git
a/xtable-core/src/test/java/org/apache/xtable/delta/TestDeltaSchemaExtractor.java
b/xtable-core/src/test/java/org/apache/xtable/delta/TestDeltaSchemaExtractor.java
index 960bc78b..a8ca9410 100644
---
a/xtable-core/src/test/java/org/apache/xtable/delta/TestDeltaSchemaExtractor.java
+++
b/xtable-core/src/test/java/org/apache/xtable/delta/TestDeltaSchemaExtractor.java
@@ -472,6 +472,98 @@ public class TestDeltaSchemaExtractor {
internalSchema,
DeltaSchemaExtractor.getInstance().toInternalSchema(structRepresentation));
}
+ @Test
+ public void testMapWithStructKey() {
+ InternalSchema structKeySchema =
+ InternalSchema.builder()
+ .name("struct")
+ .isNullable(false)
+ .fields(
+ Arrays.asList(
+ InternalField.builder()
+ .name("id")
+ .parentPath("structKeyMap._one_field_key")
+ .schema(
+ InternalSchema.builder()
+ .name("long")
+ .dataType(InternalType.LONG)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("region")
+ .parentPath("structKeyMap._one_field_key")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .isNullable(true)
+ .build())
+
.defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build()))
+ .dataType(InternalType.RECORD)
+ .build();
+ InternalSchema structValueSchema =
+ InternalSchema.builder()
+ .name("struct")
+ .isNullable(true)
+ .fields(
+ Collections.singletonList(
+ InternalField.builder()
+ .name("payload")
+ .parentPath("structKeyMap._one_field_value")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .isNullable(false)
+ .build())
+ .build()))
+ .dataType(InternalType.RECORD)
+ .build();
+ InternalSchema internalSchema =
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .isNullable(false)
+ .fields(
+ Collections.singletonList(
+ InternalField.builder()
+ .name("structKeyMap")
+ .schema(
+ InternalSchema.builder()
+ .name("map")
+ .isNullable(true)
+ .dataType(InternalType.MAP)
+ .fields(
+ Arrays.asList(
+ InternalField.builder()
+
.name(InternalField.Constants.MAP_KEY_FIELD_NAME)
+ .parentPath("structKeyMap")
+ .schema(structKeySchema)
+ .build(),
+ InternalField.builder()
+
.name(InternalField.Constants.MAP_VALUE_FIELD_NAME)
+ .parentPath("structKeyMap")
+ .schema(structValueSchema)
+ .build()))
+ .build())
+
.defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build()))
+ .build();
+
+ StructType keyStruct =
+ new StructType()
+ .add("id", DataTypes.LongType, false)
+ .add("region", DataTypes.StringType, true);
+ StructType valueStruct = new StructType().add("payload",
DataTypes.StringType, false);
+ StructType structRepresentation =
+ new StructType().add("structKeyMap",
DataTypes.createMapType(keyStruct, valueStruct, true));
+
+ Assertions.assertEquals(
+ internalSchema,
DeltaSchemaExtractor.getInstance().toInternalSchema(structRepresentation));
+ }
+
@Test
public void testLists() {
InternalSchema recordListElementSchema =
diff --git
a/xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelSchemaExtractor.java
b/xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelSchemaExtractor.java
index 6873b2cb..8c3dda57 100644
---
a/xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelSchemaExtractor.java
+++
b/xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelSchemaExtractor.java
@@ -499,6 +499,174 @@ public class TestDeltaKernelSchemaExtractor {
DeltaKernelSchemaExtractor.getInstance().toInternalSchema(structRepresentation));
}
+ @Test
+ public void testMapWithStructKey() {
+ InternalSchema structKeySchema =
+ InternalSchema.builder()
+ .name("struct")
+ .isNullable(false)
+ .fields(
+ Arrays.asList(
+ InternalField.builder()
+ .name("id")
+ .parentPath("structKeyMap._one_field_key")
+ .schema(
+ InternalSchema.builder()
+ .name("long")
+ .dataType(InternalType.LONG)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("region")
+ .parentPath("structKeyMap._one_field_key")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .isNullable(true)
+ .build())
+
.defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build()))
+ .dataType(InternalType.RECORD)
+ .build();
+ InternalSchema structValueSchema =
+ InternalSchema.builder()
+ .name("struct")
+ .isNullable(true)
+ .fields(
+ Collections.singletonList(
+ InternalField.builder()
+ .name("payload")
+ .parentPath("structKeyMap._one_field_value")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .isNullable(false)
+ .build())
+ .build()))
+ .dataType(InternalType.RECORD)
+ .build();
+ InternalSchema internalSchema =
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .isNullable(false)
+ .fields(
+ Collections.singletonList(
+ InternalField.builder()
+ .name("structKeyMap")
+ .schema(
+ InternalSchema.builder()
+ .name("map")
+ .isNullable(true)
+ .dataType(InternalType.MAP)
+ .fields(
+ Arrays.asList(
+ InternalField.builder()
+
.name(InternalField.Constants.MAP_KEY_FIELD_NAME)
+ .parentPath("structKeyMap")
+ .schema(structKeySchema)
+ .build(),
+ InternalField.builder()
+
.name(InternalField.Constants.MAP_VALUE_FIELD_NAME)
+ .parentPath("structKeyMap")
+ .schema(structValueSchema)
+ .build()))
+ .build())
+
.defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build()))
+ .build();
+
+ io.delta.kernel.types.StructType keyStruct =
+ new StructType().add("id", LongType.LONG, false).add("region",
StringType.STRING, true);
+ io.delta.kernel.types.StructType valueStruct =
+ new StructType().add("payload", StringType.STRING, false);
+ io.delta.kernel.types.StructType structRepresentation =
+ new StructType().add("structKeyMap", new MapType(keyStruct,
valueStruct, true));
+
+ Assertions.assertEquals(
+ internalSchema,
+
DeltaKernelSchemaExtractor.getInstance().toInternalSchema(structRepresentation));
+ }
+
+ @Test
+ public void testBinaryInComplexTypesDoesNotThrow() {
+ // Regression: BinaryType nested inside MapType/ArrayType receives null
originalMetadata
+ // from the parent recursion; ensure the BinaryType branch handles null
without NPE.
+ InternalSchema internalSchema =
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .isNullable(false)
+ .fields(
+ Arrays.asList(
+ InternalField.builder()
+ .name("binaryKeyMap")
+ .schema(
+ InternalSchema.builder()
+ .name("map")
+ .isNullable(true)
+ .dataType(InternalType.MAP)
+ .fields(
+ Arrays.asList(
+ InternalField.builder()
+
.name(InternalField.Constants.MAP_KEY_FIELD_NAME)
+ .parentPath("binaryKeyMap")
+ .schema(
+ InternalSchema.builder()
+ .name("binary")
+
.dataType(InternalType.BYTES)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+
.name(InternalField.Constants.MAP_VALUE_FIELD_NAME)
+ .parentPath("binaryKeyMap")
+ .schema(
+ InternalSchema.builder()
+ .name("binary")
+
.dataType(InternalType.BYTES)
+ .isNullable(true)
+ .build())
+ .build()))
+ .build())
+
.defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build(),
+ InternalField.builder()
+ .name("binaryList")
+ .schema(
+ InternalSchema.builder()
+ .name("array")
+ .isNullable(false)
+ .dataType(InternalType.LIST)
+ .fields(
+ Collections.singletonList(
+ InternalField.builder()
+
.name(InternalField.Constants.ARRAY_ELEMENT_FIELD_NAME)
+ .parentPath("binaryList")
+ .schema(
+ InternalSchema.builder()
+ .name("binary")
+
.dataType(InternalType.BYTES)
+ .isNullable(false)
+ .build())
+ .build()))
+ .build())
+ .build()))
+ .build();
+
+ io.delta.kernel.types.StructType structRepresentation =
+ new StructType()
+ .add("binaryKeyMap", new MapType(BinaryType.BINARY,
BinaryType.BINARY, true))
+ .add("binaryList", new ArrayType(BinaryType.BINARY, false), false);
+
+ Assertions.assertEquals(
+ internalSchema,
+
DeltaKernelSchemaExtractor.getInstance().toInternalSchema(structRepresentation));
+ }
+
@Test
public void testLists() {
InternalSchema recordListElementSchema =
