This is an automated email from the ASF dual-hosted git repository. XiaoHongbo-Hope pushed a commit to branch release-2.1 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 512f285d1c1c716a39b1b0939ee345a5fa379b03 Author: Xiangyi Zhu <[email protected]> AuthorDate: Fri Sep 18 11:59:38 2026 +0800 [common] Infer a variant shredding schema regardless of the writer's key order (#9940) --- .../data/variant/InferVariantShreddingSchema.java | 24 +++++++---- .../variant/InferVariantShreddingSchemaTest.java | 47 ++++++++++++++++++++++ .../writer/InferVariantShreddingWriteTest.java | 41 +++++++++++++++++++ .../apache/paimon/spark/sql/VariantTestBase.scala | 17 ++++++++ 4 files changed, 121 insertions(+), 8 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java b/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java index 7c80064c24..cf4d52e9d5 100644 --- a/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java +++ b/paimon-common/src/main/java/org/apache/paimon/data/variant/InferVariantShreddingSchema.java @@ -18,6 +18,7 @@ package org.apache.paimon.data.variant; +import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.InternalRow; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataField; @@ -313,13 +314,11 @@ public class InferVariantShreddingSchema { fields.add(dataField); } - // According to the variant spec, object fields must be sorted alphabetically - for (int i = 1; i < fields.size(); i++) { - if (fields.get(i - 1).name().compareTo(fields.get(i).name()) >= 0) { - throw new RuntimeException( - "Variant object fields must be sorted alphabetically"); - } - } + // Writers disagree on how object keys are ordered: Paimon's builder sorts them + // by UTF-8 bytes, Spark's by UTF-16 code units, which differ once a supplementary + // character meets one from U+E000 upwards. mergeRowTypes merges sorted lists, so + // put every object into the same order here instead of trusting the writer's. + fields.sort((f1, f2) -> compareKeys(f1.name(), f2.name())); return new RowType(fields); @@ -391,6 +390,15 @@ public class InferVariantShreddingSchema { } } + /** + * The one key order inference works in: UTF-8 bytes, as Paimon's own builder sorts. {@link + * String#compareTo} orders by UTF-16 code units instead, which puts a supplementary character + * such as an emoji before every character from U+E000 upwards, the reverse of the byte order. + */ + private static int compareKeys(String key1, String key2) { + return BinaryString.fromString(key1).compareTo(BinaryString.fromString(key2)); + } + private double getFieldCount(DataField field) { // Read count from description field String desc = field.description(); @@ -470,7 +478,7 @@ public class InferVariantShreddingSchema { DataField field2 = fields2.get(f2Idx); String f1Name = field1.name(); String f2Name = field2.name(); - int comp = f1Name.compareTo(f2Name); + int comp = compareKeys(f1Name, f2Name); if (comp == 0) { DataType dataType = mergeSchema(field1.type(), field2.type()); diff --git a/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java b/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java index 2b7835d7b1..51acc76f16 100644 --- a/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/data/variant/InferVariantShreddingSchemaTest.java @@ -239,6 +239,53 @@ public class InferVariantShreddingSchemaTest { .isEqualTo(variantShreddingSchema(DataTypes.VARIANT())); } + @Test + void testInferSchemaWithKeysInEitherWriterOrder() { + RowType schema = RowType.of(new DataType[] {DataTypes.VARIANT()}, new String[] {"v"}); + + // Paimon's builder sorts keys by UTF-8 bytes, where U+FFE5 (EF BF A5) precedes the + // emoji U+1F600 (F0 9F 98 80); Spark's sorts by UTF-16 code units, where the emoji's + // surrogate pair (D83D) comes first. Inference used to reject the byte order as unsorted, + // and it must merge objects written in either order. + String yen = "\uFFE5"; + String smile = new String(Character.toChars(0x1F600)); + GenericVariant byteOrdered = + GenericVariant.fromJson("{\"" + yen + "\": 100, \"" + smile + "\": \"s\"}"); + GenericVariant codeUnitOrdered = withSwappedFields(byteOrdered); + assertThat(byteOrdered.getFieldAtIndex(0).key).isEqualTo(yen); + assertThat(codeUnitOrdered.getFieldAtIndex(0).key).isEqualTo(smile); + GenericVariant third = GenericVariant.fromJson("{\"" + smile + "\": \"t\", \"a\": 2}"); + + // merged in byte order, with every key kept exactly once and both rows counted + RowType expectedType = + RowType.of( + new DataType[] {DataTypes.BIGINT(), DataTypes.BIGINT(), DataTypes.STRING()}, + new String[] {"a", yen, smile}); + for (GenericVariant first : new GenericVariant[] {byteOrdered, codeUnitOrdered}) { + List<InternalRow> rows = Arrays.asList(GenericRow.of(first), GenericRow.of(third)); + RowType inferredSchema = defaultInferVariantShreddingSchema(schema).inferSchema(rows); + assertThat(inferredSchema.getField("v").type()) + .isEqualTo(variantShreddingSchema(expectedType)); + } + } + + /** + * The same two-field object with its entries swapped, the order a UTF-16 sorting writer uses. + */ + private static GenericVariant withSwappedFields(GenericVariant twoFields) { + byte[] value = twoFields.value().clone(); + // header, size, then 2 one-byte ids and 3 one-byte offsets + int idStart = 2; + int offsetStart = idStart + 2; + byte temporary = value[idStart]; + value[idStart] = value[idStart + 1]; + value[idStart + 1] = temporary; + temporary = value[offsetStart]; + value[offsetStart] = value[offsetStart + 1]; + value[offsetStart + 1] = temporary; + return new GenericVariant(value, twoFields.metadata()); + } + @Test void testInferSchemaWithNullValues() { // Schema: row<v: variant> diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java index f1187f2b54..95d72921dc 100644 --- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java +++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/writer/InferVariantShreddingWriteTest.java @@ -496,6 +496,47 @@ public class InferVariantShreddingWriteTest { assertThat(result2.get(1)).isEqualTo(GenericRow.of(GenericRow.of(5L))); } + @Test + public void testInferSchemaWithKeysOrderedByUtf8Bytes() throws Exception { + ParquetFileFormat format = createFormat(); + RowType writeType = DataTypes.ROW(DataTypes.FIELD(0, "v", DataTypes.VARIANT())); + + // U+FFE5 sorts before the emoji U+1F600 by UTF-8 bytes but after it by UTF-16 code + // units; inference used to reject such an object as unsorted and fail the file write + String yen = "\uFFE5"; + String smile = new String(Character.toChars(0x1F600)); + FormatWriterFactory factory = format.createWriterFactory(writeType); + writeRows( + factory, + GenericRow.of( + GenericVariant.fromJson("{\"" + yen + "\":100,\"" + smile + "\":\"s\"}")), + GenericRow.of(GenericVariant.fromJson("{\"" + smile + "\":\"t\",\"a\":2}"))); + + RowType expectShreddedType = + RowType.of( + new DataType[] {DataTypes.BIGINT(), DataTypes.BIGINT(), DataTypes.STRING()}, + new String[] {"a", yen, smile}); + verifyShreddingSchema(expectShreddedType); + + List<InternalRow> result = readRows(format, writeType); + assertThat(result.get(0).getVariant(0).toJson()) + .isEqualTo("{\"" + yen + "\":100,\"" + smile + "\":\"s\"}"); + assertThat(result.get(1).getVariant(0).toJson()) + .isEqualTo("{\"a\":2,\"" + smile + "\":\"t\"}"); + + RowType variantRowType = + VariantMetadataUtils.VariantRowTypeBuilder.builder() + .field(DataTypes.STRING(), "$['" + smile + "']") + .field(DataTypes.BIGINT(), "$['" + yen + "']") + .build(); + RowType readType = DataTypes.ROW(DataTypes.FIELD(0, "v", variantRowType)); + List<InternalRow> result2 = readRows(format, readType); + assertThat(result2.get(0)) + .isEqualTo(GenericRow.of(GenericRow.of(BinaryString.fromString("s"), 100L))); + assertThat(result2.get(1)) + .isEqualTo(GenericRow.of(GenericRow.of(BinaryString.fromString("t"), null))); + } + @Test public void testInferSchemaWithNullValues() throws Exception { ParquetFileFormat format = createFormat(); diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala index 4501713510..6bddc24fd4 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/VariantTestBase.scala @@ -459,6 +459,23 @@ abstract class VariantTestBase extends PaimonSparkTestBase { ) } + test("Paimon Variant: keys that UTF-8 and UTF-16 order differently under inferred shredding") { + sql("CREATE TABLE T (id INT, v VARIANT)") + // Spark orders "\uFFE5" after the emoji (UTF-16 code units), Paimon's builder before it + // (UTF-8 bytes); inferred shredding used to reject one of the two orders as unsorted and + // fail the write. Key order is not asserted because each writer renders its own. + sql("""INSERT INTO T VALUES + | (1, parse_json('{"\uFFE5":100,"\uD83D\uDE00":"s"}')), + | (2, parse_json('{"\uD83D\uDE00":"t","a":2}')) + |""".stripMargin) + + checkAnswer( + sql( + "SELECT id, variant_get(v, '$[\"\uD83D\uDE00\"]', 'string'), try_variant_get(v, '$[\"\uFFE5\"]', 'bigint'), try_variant_get(v, '$.a', 'int') FROM T ORDER BY id"), + Seq(Row(1, "s", 100L, null), Row(2, "t", null, 2)) + ) + } + test("Paimon Variant: read and write variant with null value") { withTable("source_tbl", "target_tbl") { sql("CREATE TABLE source_tbl (id INT, js STRING) USING paimon")
