This is an automated email from the ASF dual-hosted git repository. XiaoHongbo-Hope pushed a commit to branch release-2.1 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit eccfcb4386e86563138f1a3fe40568ddb43c8ace Author: Xiangyi Zhu <[email protected]> AuthorDate: Thu Sep 17 17:06:10 2026 +0800 [common] Keep blank variant object keys unshredded during schema inference (#9813) --- .../data/variant/InferVariantShreddingSchema.java | 8 +++++- .../variant/InferVariantShreddingSchemaTest.java | 28 +++++++++++++++++++ .../writer/InferVariantShreddingWriteTest.java | 31 ++++++++++++++++++++++ .../apache/paimon/spark/sql/VariantTestBase.scala | 14 ++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java b/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java index 318266eb67..7c80064c24 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java @@ -26,6 +26,7 @@ import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.RowType; import org.apache.paimon.types.VariantType; +import org.apache.paimon.utils.StringUtils; import java.math.BigDecimal; import java.util.ArrayList; @@ -301,6 +302,11 @@ public class InferVariantShreddingSchema { for (int i = 0; i < size; i++) { GenericVariant.ObjectField field = v.getFieldAtIndex(i); + if (StringUtils.isNullOrWhitespaceOnly(field.key)) { + // A blank key is a valid variant object key but not a RowType field + // name, so leave it in the unshredded value instead of failing the write + continue; + } DataType fieldType = schemaOf(field.value, maxDepth - 1); // Store count in description temporarily (will be used in mergeRowTypes) DataField dataField = new DataField(i, field.key, fieldType, "1"); @@ -308,7 +314,7 @@ public class InferVariantShreddingSchema { } // According to the variant spec, object fields must be sorted alphabetically - for (int i = 1; i < size; i++) { + for (int i = 1; i < fields.size(); i++) { if (fields.get(i - 1).name().compareTo(fields.get(i).name()) >= 0) { throw new RuntimeException( "Variant object fields must be sorted alphabetically"); diff --git a/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java b/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java index c1f8628009..2b7835d7b1 100644 --- a/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java @@ -31,6 +31,7 @@ import java.math.BigDecimal; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -211,6 +212,33 @@ public class InferVariantShreddingSchemaTest { .isEqualTo(variantShreddingSchema(expectedType)); } + @Test + void testInferSchemaSkipsBlankKeys() { + RowType schema = RowType.of(new DataType[] {DataTypes.VARIANT()}, new String[] {"v"}); + + // An empty or blank key is a valid variant object key but not a RowType field name; + // it must stay in the unshredded value rather than fail the inference + GenericVariant variant1 = GenericVariant.fromJson("{\"\": 1, \" \": 2, \"a\": 3}"); + GenericVariant variant2 = GenericVariant.fromJson("{\"\": 4, \"a\": 5}"); + + List<InternalRow> rows = Arrays.asList(GenericRow.of(variant1), GenericRow.of(variant2)); + + InferVariantShreddingSchema inferrer = defaultInferVariantShreddingSchema(schema); + RowType inferredSchema = inferrer.inferSchema(rows); + + RowType expectedType = RowType.of(new DataType[] {DataTypes.BIGINT()}, new String[] {"a"}); + assertThat(inferredSchema.getField("v").type()) + .isEqualTo(variantShreddingSchema(expectedType)); + + // an object made only of blank keys has nothing to shred and stays unshredded + RowType blankOnly = + inferrer.inferSchema( + Collections.singletonList( + GenericRow.of(GenericVariant.fromJson("{\"\": 1}")))); + assertThat(blankOnly.getField("v").type()) + .isEqualTo(variantShreddingSchema(DataTypes.VARIANT())); + } + @Test void testInferSchemaWithNullValues() { // Schema: row<v: variant> diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java index f59ffa462e..f1187f2b54 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java @@ -465,6 +465,37 @@ public class InferVariantShreddingWriteTest { assertThat(result2.get(1)).isEqualTo(GenericRow.of(GenericRow.of(20.5, 7L))); } + @Test + public void testInferSchemaSkipsBlankKeys() throws Exception { + ParquetFileFormat format = createFormat(); + RowType writeType = DataTypes.ROW(DataTypes.FIELD(0, "v", DataTypes.VARIANT())); + + // an empty key is a valid variant object key but not a RowType field name; it used + // to fail the whole file write and must stay in the unshredded value instead + FormatWriterFactory factory = format.createWriterFactory(writeType); + writeRows( + factory, + GenericRow.of(GenericVariant.fromJson("{\"\":1,\"a\":2}")), + GenericRow.of(GenericVariant.fromJson("{\"\":3,\" \":4,\"a\":5}"))); + + RowType expectShreddedType = + RowType.of(new DataType[] {DataTypes.BIGINT()}, new String[] {"a"}); + verifyShreddingSchema(expectShreddedType); + + List<InternalRow> result = readRows(format, writeType); + assertThat(result.get(0).getVariant(0).toJson()).isEqualTo("{\"\":1,\"a\":2}"); + assertThat(result.get(1).getVariant(0).toJson()).isEqualTo("{\"\":3,\" \":4,\"a\":5}"); + + RowType variantRowType = + VariantMetadataUtils.VariantRowTypeBuilder.builder() + .field(DataTypes.BIGINT(), "$.a") + .build(); + RowType readType = DataTypes.ROW(DataTypes.FIELD(0, "v", variantRowType)); + List<InternalRow> result2 = readRows(format, readType); + assertThat(result2.get(0)).isEqualTo(GenericRow.of(GenericRow.of(2L))); + assertThat(result2.get(1)).isEqualTo(GenericRow.of(GenericRow.of(5L))); + } + @Test public void testInferSchemaWithNullValues() throws Exception { ParquetFileFormat format = createFormat(); diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala index 70b58e0fe9..0b50b12870 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala @@ -445,6 +445,20 @@ abstract class VariantTestBase extends PaimonSparkTestBase { Seq(Row(1, 10.0, 100L), Row(2, 20.5, 7L))) } + test("Paimon Variant: blank object keys under inferred shredding") { + sql("CREATE TABLE T (id INT, v VARIANT)") + // an empty key cannot become a shredded column and used to fail inferred-shredding writes + sql("""INSERT INTO T VALUES + | (1, parse_json('{"":1,"a":2}')), + | (2, parse_json('{"":3," ":4,"a":5}')) + |""".stripMargin) + + checkAnswer( + sql("SELECT id, to_json(v), variant_get(v, '$.a', 'bigint') FROM T ORDER BY id"), + Seq(Row(1, """{"":1,"a":2}""", 2L), Row(2, """{"":3," ":4,"a":5}""", 5L)) + ) + } + test("Paimon Variant: read and write variant with null value") { withTable("source_tbl", "target_tbl") { sql("CREATE TABLE source_tbl (id INT, js STRING) USING paimon")
