This is an automated email from the ASF dual-hosted git repository.
vinishjail97 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-xtable.git
The following commit(s) were added to refs/heads/main by this push:
new 39d6cb45 [958] Mark only non-required parquet groups nullable, never
the root message (#989)
39d6cb45 is described below
commit 39d6cb45e5cd1cb22748677bbe0a48f6835d04ff
Author: Sylwester Lachiewicz <[email protected]>
AuthorDate: Thu Oct 29 21:58:24 2026 +0100
[958] Mark only non-required parquet groups nullable, never the root
message (#989)
Parquet gives the root MessageType the repetition REPEATED, so requiring a
nullable child as well made the root record nullable whenever one column was
optional, and Hudi then failed to build column stats on the ["null", record]
union. The same condition reported an optional group whose fields are all
required as non-nullable. A group's nullability is its repetition alone.
Co-authored-by: Vinish Reddy <[email protected]>
---
.../xtable/parquet/ParquetSchemaExtractor.java | 6 ++--
.../xtable/parquet/ITParquetConversionSource.java | 5 ++--
.../xtable/parquet/TestParquetSchemaExtractor.java | 33 ++++++++++++++++++++++
3 files changed, 39 insertions(+), 5 deletions(-)
diff --git
a/xtable-core/src/main/java/org/apache/xtable/parquet/ParquetSchemaExtractor.java
b/xtable-core/src/main/java/org/apache/xtable/parquet/ParquetSchemaExtractor.java
index 48144287..2cc1650b 100644
---
a/xtable-core/src/main/java/org/apache/xtable/parquet/ParquetSchemaExtractor.java
+++
b/xtable-core/src/main/java/org/apache/xtable/parquet/ParquetSchemaExtractor.java
@@ -30,6 +30,7 @@ import lombok.NoArgsConstructor;
import org.apache.avro.Schema;
import org.apache.parquet.schema.LogicalTypeAnnotation;
+import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.PrimitiveType;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.apache.parquet.schema.Type;
@@ -319,9 +320,8 @@ public class ParquetSchemaExtractor {
// RECORD Type (non-nullable elements)
if (!schema.asGroupType().getName().equals("list")
&& !Arrays.asList("key_value",
"map").contains(schema.asGroupType().getName())) {
- boolean isNullable =
- subFields.stream().anyMatch(ele -> ele.getSchema().isNullable())
- && isNullable(schema.asGroupType());
+ // The root message is the row itself and is never null; parquet
marks it REPEATED.
+ boolean isNullable = !(schema instanceof MessageType) &&
isNullable(schema);
return InternalSchema.builder()
.name(schema.getName())
.comment(null)
diff --git
a/xtable-core/src/test/java/org/apache/xtable/parquet/ITParquetConversionSource.java
b/xtable-core/src/test/java/org/apache/xtable/parquet/ITParquetConversionSource.java
index cededf35..66773f2a 100644
---
a/xtable-core/src/test/java/org/apache/xtable/parquet/ITParquetConversionSource.java
+++
b/xtable-core/src/test/java/org/apache/xtable/parquet/ITParquetConversionSource.java
@@ -228,7 +228,7 @@ public class ITParquetConversionSource {
RowFactory.create(
3, "Charlie", true, 35.2, new
Timestamp(System.currentTimeMillis() + 2000)),
RowFactory.create(
- 4, "David", false, 29.5, new
Timestamp(System.currentTimeMillis() + 3000)),
+ 4, null, false, 29.5, new Timestamp(System.currentTimeMillis()
+ 3000)),
RowFactory.create(
5, "Eve", true, 22.2, new Timestamp(System.currentTimeMillis()
+ 4000)));
@@ -236,7 +236,8 @@ public class ITParquetConversionSource {
DataTypes.createStructType(
new StructField[] {
DataTypes.createStructField("id", DataTypes.IntegerType, false),
- DataTypes.createStructField("name", DataTypes.StringType, false),
+ // nullable, as Spark writes most columns; covers #958
+ DataTypes.createStructField("name", DataTypes.StringType, true),
DataTypes.createStructField("hasSiblings",
DataTypes.BooleanType, false),
DataTypes.createStructField("age", DataTypes.DoubleType, false),
DataTypes.createStructField(
diff --git
a/xtable-core/src/test/java/org/apache/xtable/parquet/TestParquetSchemaExtractor.java
b/xtable-core/src/test/java/org/apache/xtable/parquet/TestParquetSchemaExtractor.java
index 8dbceaa5..13282198 100644
---
a/xtable-core/src/test/java/org/apache/xtable/parquet/TestParquetSchemaExtractor.java
+++
b/xtable-core/src/test/java/org/apache/xtable/parquet/TestParquetSchemaExtractor.java
@@ -396,6 +396,39 @@ public class TestParquetSchemaExtractor {
Assertions.assertEquals(internalSchema,
schemaExtractor.toInternalSchema(messageType, null));
}
+ static Stream<Arguments> recordNullability() {
+ return Stream.of(
+ Arguments.of(
+ "root message with an optional column",
+ Types.buildMessage()
+ .required(PrimitiveTypeName.INT64)
+ .named("id")
+ .optional(PrimitiveTypeName.INT32)
+ .named("name")
+ .named("my_record"),
+ false),
+ Arguments.of(
+ "optional group with only required fields",
+
Types.optionalGroup().required(PrimitiveTypeName.INT64).named("id").named("my_group"),
+ true),
+ Arguments.of(
+ "optional group with an optional field",
+
Types.optionalGroup().optional(PrimitiveTypeName.INT64).named("id").named("my_group"),
+ true),
+ Arguments.of(
+ "required group with an optional field",
+
Types.requiredGroup().optional(PrimitiveTypeName.INT64).named("id").named("my_group"),
+ false));
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("recordNullability")
+ void testRecordNullability(String description, GroupType parquetSchema,
boolean nullable) {
+ InternalSchema internalSchema =
schemaExtractor.toInternalSchema(parquetSchema, null);
+ Assertions.assertEquals(InternalType.RECORD, internalSchema.getDataType());
+ Assertions.assertEquals(nullable, internalSchema.isNullable());
+ }
+
static Stream<Arguments> listEncodings() {
return Stream.of(
Arguments.of(
