This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20638-3e3a4955d33ca8dd6efa2216c76ea88056afa61f in repository https://gitbox.apache.org/repos/asf/hudi.git
commit e8a076d08d5875c404736941ae88c9bbc7449b82 Author: Alexander Narsudinov <[email protected]> AuthorDate: Thu Nov 19 07:52:13 2026 +0000 fix(trino): preserve leading whitespace when serializing CHAR values (#20638) * fix(trino): preserve leading whitespace when serializing CHAR values The Avro-to-page path uses String.trim for CHAR values. That removes significant leading whitespace and trailing control characters, while failing to enforce the declared CHAR length. Base-only Parquet reads and merged reads can therefore return different values for the same logical column. Use Trino's truncateToLengthAndTrimSpaces helper with the declared CharType. It truncates by Unicode code points and removes the trailing ASCII-space padding required by Trino's native CHAR representation. Add focused cases for leading spaces, all-space values, tabs, a supplementary Unicode character and overlength input. Exercise both Java String and Avro Utf8 representations for every case. Validation: all 5 TestHudiAvroCharSerialization cases passed with this fix alone on a9185b7c4e83. * fix(trino): enforce VARCHAR bounds and test CHAR snapshot reads Truncate bounded VARCHAR values with Trino's helper while preserving unbounded values and whitespace. Clarify the CHAR conversion comment. Move CHAR cases into TestHudiAvroSerializer and add overlength Unicode coverage. Add MOR regression tests for base/snapshot parity, leading-space filters, and tabs in updated rows using a CHAR(5) fixture column. * test(trino): drop redundant CHAR emoji case and escape emoji Remove the in-length emoji CHAR case, which ' ab ' already covers, and write the remaining emoji cases with \uD83D\uDE00 escapes so the test source stays ASCII. * refactor(trino): rely on truncateToLength for unbounded VARCHAR Varchars.truncateToLength(Slice, VarcharType) already returns the slice unchanged for an unbounded VARCHAR, so drop the extra isUnbounded() check and note which reader the truncation matches. --------- Co-authored-by: voon <[email protected]> --- .../trino/plugin/hudi/util/HudiAvroSerializer.java | 23 ++++------ .../plugin/hudi/TestHudiMorMergeModeSemantics.java | 24 +++++++++++ .../CommitTimeOrderingHudiTablesInitializer.java | 14 ++++--- .../plugin/hudi/util/TestHudiAvroSerializer.java | 49 ++++++++++++++++++++++ 4 files changed, 91 insertions(+), 19 deletions(-) diff --git a/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java index cb2414ee40d8..68bc8dce4d53 100644 --- a/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java +++ b/hudi-trino/src/main/java/io/trino/plugin/hudi/util/HudiAvroSerializer.java @@ -67,6 +67,7 @@ import static io.trino.plugin.hudi.HudiUtil.getFieldFromSchema; import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; import static io.trino.spi.StandardErrorCode.NUMERIC_VALUE_OUT_OF_RANGE; import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.Chars.truncateToLengthAndTrimSpaces; import static io.trino.spi.type.DateType.DATE; import static io.trino.spi.type.Decimals.writeBigDecimal; import static io.trino.spi.type.Decimals.writeShortDecimal; @@ -83,6 +84,7 @@ import static io.trino.spi.type.Timestamps.MICROSECONDS_PER_SECOND; import static io.trino.spi.type.Timestamps.NANOSECONDS_PER_MICROSECOND; import static io.trino.spi.type.Timestamps.PICOSECONDS_PER_MICROSECOND; import static io.trino.spi.type.TinyintType.TINYINT; +import static io.trino.spi.type.Varchars.truncateToLength; import static java.lang.Integer.parseInt; import static java.lang.Math.floorDiv; import static java.lang.Math.floorMod; @@ -435,16 +437,9 @@ public class HudiAvroSerializer private static void writeSlice(BlockBuilder output, Type type, Object value) { - if (type instanceof VarcharType) { - if (value instanceof Utf8) { - type.writeSlice(output, utf8Slice(((Utf8) value).toString())); - } - else if (value instanceof String) { - type.writeSlice(output, utf8Slice((String) value)); - } - else { - type.writeSlice(output, utf8Slice(value.toString())); - } + if (type instanceof VarcharType varcharType) { + // Match Trino's Parquet VARCHAR reader: truncate a bounded VARCHAR to its length in code points + type.writeSlice(output, truncateToLength(utf8Slice(value.toString()), varcharType)); } else if (type instanceof VarbinaryType) { if (value instanceof ByteBuffer) { @@ -458,7 +453,7 @@ public class HudiAvroSerializer format("Unhandled type for %s: %s | value type: %s", type.getJavaType().getSimpleName(), type, value.getClass().getName())); } } - else if (type instanceof CharType) { + else if (type instanceof CharType charType) { String stringValue; if (value instanceof Utf8) { stringValue = ((Utf8) value).toString(); @@ -470,9 +465,9 @@ public class HudiAvroSerializer // Fallback: convert any other object to its string representation stringValue = value.toString(); } - // IMPORTANT: Char types may be padded with trailing "space" characters to make up for length if the contents are lesser than defined length. - // Need to trim out trailing spaces as Slice representing Char should not have trailing spaces - type.writeSlice(output, utf8Slice(stringValue.trim())); + // Match Trino's Parquet CHAR reader: truncate to the declared length in code points, then strip + // trailing U+0020 padding only, keeping leading whitespace and trailing control characters + type.writeSlice(output, truncateToLengthAndTrimSpaces(utf8Slice(stringValue), charType)); } else { throw new TrinoException(GENERIC_INTERNAL_ERROR, diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMorMergeModeSemantics.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMorMergeModeSemantics.java index a3e62db9de0d..fb4bc50ad3a0 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMorMergeModeSemantics.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/TestHudiMorMergeModeSemantics.java @@ -116,6 +116,30 @@ public class TestHudiMorMergeModeSemantics "VALUES ('k1', 'k1_updated', CAST(11 AS BIGINT)), ('k3', 'k3_base', 30)"); } + @Test + public void testCharMatchesBaseReaderOnUntouchedRow() + { + String query = "SELECT code FROM %s WHERE key = 'k3'"; + Object readOptimized = computeScalar(query.formatted(CommitTimeOrderingHudiTablesInitializer.TABLE_NAME)); + assertThat(readOptimized).isEqualTo(" ab "); + assertThat(computeScalar(query.formatted(CommitTimeOrderingHudiTablesInitializer.RT_TABLE_NAME))).isEqualTo(readOptimized); + } + + @Test + public void testCharFilterMatchesLeadingSpaceOnSnapshotRead() + { + assertQuery( + "SELECT key FROM " + CommitTimeOrderingHudiTablesInitializer.RT_TABLE_NAME + " WHERE code = CAST(' ab' AS CHAR(5))", + "VALUES 'k3'"); + } + + @Test + public void testCharKeepsTabsOnUpdatedRow() + { + assertThat(computeScalar("SELECT code FROM " + CommitTimeOrderingHudiTablesInitializer.RT_TABLE_NAME + " WHERE key = 'k1'")) + .isEqualTo("\tx\t "); + } + @Test public void testCountAfterDeletes() { diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CommitTimeOrderingHudiTablesInitializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CommitTimeOrderingHudiTablesInitializer.java index 8fa5c7fb3d79..f72e0053973d 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CommitTimeOrderingHudiTablesInitializer.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/testing/CommitTimeOrderingHudiTablesInitializer.java @@ -33,6 +33,7 @@ import java.util.Optional; import static io.trino.metastore.HiveType.HIVE_LONG; import static io.trino.metastore.HiveType.HIVE_STRING; +import static io.trino.plugin.hudi.testing.TypeInfoHelper.charHiveType; /** * Creates a non-partitioned Merge-On-Read table in {@link RecordMergeMode#COMMIT_TIME_ORDERING} that @@ -63,6 +64,7 @@ public class CommitTimeOrderingHudiTablesInitializer return ImmutableList.of( new Column(RECORD_KEY_FIELD, HIVE_STRING, Optional.empty(), Map.of()), new Column("name", HIVE_STRING, Optional.empty(), Map.of()), + new Column("code", charHiveType(5), Optional.empty(), Map.of()), new Column("value", HIVE_LONG, Optional.empty(), Map.of()), new Column(ORDERING_FIELD, HIVE_LONG, Optional.empty(), Map.of())); } @@ -73,6 +75,7 @@ public class CommitTimeOrderingHudiTablesInitializer List<Schema.Field> fields = ImmutableList.of( new Schema.Field(RECORD_KEY_FIELD, Schema.create(Schema.Type.STRING)), new Schema.Field("name", Schema.create(Schema.Type.STRING)), + new Schema.Field("code", Schema.create(Schema.Type.STRING)), new Schema.Field("value", Schema.create(Schema.Type.LONG)), new Schema.Field(ORDERING_FIELD, Schema.create(Schema.Type.LONG))); return Schema.createRecord(TABLE_NAME, null, null, false, new ArrayList<>(fields)); @@ -97,9 +100,9 @@ public class CommitTimeOrderingHudiTablesInitializer // First commit: base parquet file with 3 keys at ts 100. String firstCommit = client.startCommit(); List<WriteStatus> firstStatuses = client.bulkInsert(ImmutableList.of( - record(schema, "k1", "k1_base", 10L, 100L), - record(schema, "k2", "k2_base", 20L, 100L), - record(schema, "k3", "k3_base", 30L, 100L)), firstCommit); + record(schema, "k1", "k1_base", "k1", 10L, 100L), + record(schema, "k2", "k2_base", "k2", 20L, 100L), + record(schema, "k3", "k3_base", " ab", 30L, 100L)), firstCommit); client.commit(firstCommit, firstStatuses); // Second commit (log file): update k1 with a LOWER ts (50). Commit-time ordering keeps the @@ -107,7 +110,7 @@ public class CommitTimeOrderingHudiTablesInitializer // case, which discriminates OverwriteWithLatestMerger from event-time merging. String secondCommit = client.startCommit(); List<WriteStatus> secondStatuses = client.upsert(ImmutableList.of( - record(schema, "k1", "k1_updated", 11L, 50L)), secondCommit); + record(schema, "k1", "k1_updated", "\tx\t", 11L, 50L)), secondCommit); client.commit(secondCommit, secondStatuses); // Third commit: hard delete of k2 (commit-time deletes always win). @@ -117,11 +120,12 @@ public class CommitTimeOrderingHudiTablesInitializer client.commit(deleteCommit, deleteStatuses); } - private static HoodieRecord<HoodieAvroPayload> record(Schema schema, String key, String name, long value, long ts) + private static HoodieRecord<HoodieAvroPayload> record(Schema schema, String key, String name, String code, long value, long ts) { GenericRecord record = new GenericData.Record(schema); record.put(RECORD_KEY_FIELD, key); record.put("name", name); + record.put("code", code); record.put("value", value); record.put(ORDERING_FIELD, ts); return avroRecord(record, key); diff --git a/hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestHudiAvroSerializer.java b/hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestHudiAvroSerializer.java index 514218d48add..f8cdcda1707f 100644 --- a/hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestHudiAvroSerializer.java +++ b/hudi-trino/src/test/java/io/trino/plugin/hudi/util/TestHudiAvroSerializer.java @@ -24,12 +24,15 @@ import io.trino.spi.SplitWeight; import io.trino.spi.block.Block; import io.trino.spi.block.BlockBuilder; import io.trino.spi.predicate.TupleDomain; +import io.trino.spi.type.CharType; import io.trino.spi.type.DecimalType; +import io.trino.spi.type.VarcharType; import org.apache.avro.Conversions; import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; import org.apache.avro.generic.GenericData; +import org.apache.avro.util.Utf8; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -48,6 +51,52 @@ import static org.assertj.core.api.Assertions.assertThat; class TestHudiAvroSerializer { + @ParameterizedTest + @MethodSource("charValues") + void testAppendChar(String input, String expected) + { + CharType type = CharType.createCharType(5); + for (Object value : List.of(input, new Utf8(input))) { + BlockBuilder builder = type.createBlockBuilder(null, 1); + HudiAvroSerializer.appendTo(type, value, builder); + assertThat(type.getSlice(builder.build(), 0).toStringUtf8()).isEqualTo(expected); + } + } + + private static Stream<Arguments> charValues() + { + return Stream.of( + Arguments.of(" ab ", " ab"), + Arguments.of(" ", ""), + Arguments.of("\tab\t ", "\tab\t"), + Arguments.of(" a\uD83D\uDE00\uD83D\uDE00\uD83D\uDE00x", " a\uD83D\uDE00\uD83D\uDE00\uD83D\uDE00"), + Arguments.of(" abcdef", " abcd")); + } + + @ParameterizedTest + @MethodSource("varcharValues") + void testAppendVarchar(VarcharType type, String input, String expected) + { + for (Object value : List.of(input, new Utf8(input), new StringBuilder(input))) { + BlockBuilder builder = type.createBlockBuilder(null, 1); + HudiAvroSerializer.appendTo(type, value, builder); + assertThat(type.getSlice(builder.build(), 0).toStringUtf8()).isEqualTo(expected); + } + } + + private static Stream<Arguments> varcharValues() + { + VarcharType boundedType = VarcharType.createVarcharType(5); + return Stream.of( + Arguments.of(boundedType, " ab ", " ab "), + Arguments.of(boundedType, "\tab\t ", "\tab\t "), + Arguments.of(boundedType, " abcdef", " abcd"), + Arguments.of(boundedType, " a\uD83D\uDE00\uD83D\uDE00\uD83D\uDE00x", " a\uD83D\uDE00\uD83D\uDE00\uD83D\uDE00"), + Arguments.of(boundedType, "", ""), + Arguments.of(VARCHAR, " abcdef ", " abcdef "), + Arguments.of(VARCHAR, " a\uD83D\uDE00\uD83D\uDE00\uD83D\uDE00x ", " a\uD83D\uDE00\uD83D\uDE00\uD83D\uDE00x ")); + } + /** * A short decimal is stored in Trino as the unscaled value, which is exactly what Avro writes into the * fixed bytes, so the read is a plain big-endian two's complement decode. The cases below pin the parts
