This is an automated email from the ASF dual-hosted git repository. voonhous pushed a commit to branch nada.attia/ri-bootstrap-binary-keys-oss in repository https://gitbox.apache.org/repos/asf/hudi.git
commit c7d6fc7ff0190c26283412dcf2ac0db7a6bce51c Author: voon <[email protected]> AuthorDate: Fri Jul 24 19:15:28 2026 +0800 Address review: sort Flink MDT bulk-insert keys by UTF-8 bytes, re-sort filterRowKeys candidates, document unpaired-surrogate caveat --- .../main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java | 5 ++++- .../org/apache/hudi/io/storage/HoodieNativeAvroHFileReader.java | 8 ++++++-- .../src/main/java/org/apache/hudi/common/util/StringUtils.java | 3 +++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java index 228209cf1df6..fda126214625 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java @@ -35,6 +35,7 @@ import org.apache.hudi.common.model.TableServiceType; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieNotSupportedException; import org.apache.hudi.index.FlinkHoodieIndexFactory; @@ -323,7 +324,9 @@ public class HoodieFlinkWriteClient<T> Map<String, List<HoodieRecord<T>>> preppedRecordsByFileId = preppedRecords.stream().parallel() .collect(Collectors.groupingBy(r -> r.getCurrentLocation().getFileId())); return preppedRecordsByFileId.values().stream().parallel().map(records -> { - records.sort(Comparator.comparing(HoodieRecord::getRecordKey)); + // Only used for the metadata table, whose base files are HFiles ordered by raw UTF-8 bytes, + // so sort by UTF-8 bytes rather than String (UTF-16) order for non-ASCII / binary keys. + records.sort(Comparator.comparing(HoodieRecord::getRecordKey, StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR)); HoodieWriteMetadata<List<WriteStatus>> result; BucketInfo bucketInfo = new BucketInfo(BucketType.INSERT, records.get(0).getCurrentLocation().getFileId(), records.get(0).getPartitionPath()); try (AutoCloseableWriteHandle closeableHandle = new AutoCloseableWriteHandle(bucketInfo, records.iterator(), instantTime, table, true)) { diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieNativeAvroHFileReader.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieNativeAvroHFileReader.java index 69b6dd9fd0ed..c5c7d20bbe33 100644 --- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieNativeAvroHFileReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieNativeAvroHFileReader.java @@ -27,6 +27,7 @@ import org.apache.hudi.common.model.HoodieRecordLocation; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaField; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.common.util.collection.ClosableIterator; import org.apache.hudi.common.util.collection.CloseableMappingIterator; @@ -134,8 +135,11 @@ public class HoodieNativeAvroHFileReader extends HoodieAvroHFileReaderImplBase { public Set<Pair<String, Long>> filterRowKeys(Set<String> candidateRowKeys) { try (HFileReader reader = readerFactory.createHFileReader()) { reader.seekTo(); - // candidateRowKeys must be sorted - return (candidateRowKeys instanceof TreeSet ? candidateRowKeys : new TreeSet<>(candidateRowKeys)) + // candidateRowKeys must be sorted by UTF-8 bytes to match HFile ordering because the reader + // only seeks forward. + TreeSet<String> sortedRowKeys = new TreeSet<>(StringUtils.UTF8_LEXICOGRAPHIC_COMPARATOR); + sortedRowKeys.addAll(candidateRowKeys); + return sortedRowKeys .stream() .filter(k -> { try { diff --git a/hudi-io/src/main/java/org/apache/hudi/common/util/StringUtils.java b/hudi-io/src/main/java/org/apache/hudi/common/util/StringUtils.java index c0fd685b0d3f..4b064b4c7378 100644 --- a/hudi-io/src/main/java/org/apache/hudi/common/util/StringUtils.java +++ b/hudi-io/src/main/java/org/apache/hudi/common/util/StringUtils.java @@ -137,6 +137,9 @@ public class StringUtils { * <p>Neither argument may be {@code null}; like {@link String#compareTo(String)}, a {@code null} * argument throws {@link NullPointerException}. * + * <p>Assumes well-formed UTF-16 input: {@code String#getBytes(UTF_8)} replaces unpaired surrogates + * with {@code '?'}, so strings differing only in unpaired surrogates compare equal. + * * <p>Note: encodes both strings to UTF-8 on every call; for very large sorts consider * pre-encoding keys to byte arrays once and comparing those. */
