This is an automated email from the ASF dual-hosted git repository.
laskoviymishka pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new b7e6af325f Core: Order Variant metadata dictionary and object fields
by UTF-8 byte order (#17726)
b7e6af325f is described below
commit b7e6af325fc7b372fc707ef24b760ac64e706954
Author: Neelesh Salian <[email protected]>
AuthorDate: Mon Oct 5 12:04:46 2026 -0700
Core: Order Variant metadata dictionary and object fields by UTF-8 byte
order (#17726)
* Core: Order Variant field names by unsigned UTF-8 byte order per
VariantEncoding.md
---
.../iceberg/variants/SerializedMetadata.java | 1 +
.../apache/iceberg/variants/VariantMetadata.java | 5 ++
.../org/apache/iceberg/variants/VariantUtil.java | 42 ++++++++----
.../apache/iceberg/variants/VariantTestUtil.java | 10 ++-
.../apache/iceberg/variants/ShreddedObject.java | 5 +-
.../java/org/apache/iceberg/variants/Variants.java | 2 +-
.../iceberg/variants/TestShreddedObject.java | 21 ++++++
.../variants/TestVariantMetadataFieldOrdering.java | 74 ++++++++++++++++++++++
.../iceberg/connect/data/RecordConverter.java | 4 +-
.../iceberg/connect/data/TestRecordConverter.java | 17 +++++
.../org/apache/iceberg/parquet/ParquetMetrics.java | 2 +-
.../apache/iceberg/parquet/TestVariantMetrics.java | 32 ++++++++++
12 files changed, 194 insertions(+), 21 deletions(-)
diff --git
a/api/src/main/java/org/apache/iceberg/variants/SerializedMetadata.java
b/api/src/main/java/org/apache/iceberg/variants/SerializedMetadata.java
index 49f5b39b38..85b53631ad 100644
--- a/api/src/main/java/org/apache/iceberg/variants/SerializedMetadata.java
+++ b/api/src/main/java/org/apache/iceberg/variants/SerializedMetadata.java
@@ -112,6 +112,7 @@ class SerializedMetadata implements VariantMetadata,
Serialized, Serializable {
public int id(String name) {
if (name != null) {
if (isSorted) {
+ // find retries in UTF-16 order so dictionaries written before the
UTF-8 fix still resolve
return VariantUtil.find(dict.length, name, this::get);
} else {
for (int id = 0; id < dict.length; id += 1) {
diff --git a/api/src/main/java/org/apache/iceberg/variants/VariantMetadata.java
b/api/src/main/java/org/apache/iceberg/variants/VariantMetadata.java
index a5569c9752..b4b727b9e8 100644
--- a/api/src/main/java/org/apache/iceberg/variants/VariantMetadata.java
+++ b/api/src/main/java/org/apache/iceberg/variants/VariantMetadata.java
@@ -19,10 +19,15 @@
package org.apache.iceberg.variants;
import java.nio.ByteBuffer;
+import java.util.Comparator;
import java.util.NoSuchElementException;
+import org.apache.iceberg.types.Comparators;
/** A variant metadata dictionary. */
public interface VariantMetadata {
+ /** Unsigned UTF-8 byte order for field names, per VariantEncoding.md's
sorted_strings rule. */
+ Comparator<CharSequence> FIELD_NAME_ORDER = Comparators.charSequences();
+
/** Returns the ID for a {@code name} in the dictionary, or -1 if not
present. */
int id(String name);
diff --git a/api/src/main/java/org/apache/iceberg/variants/VariantUtil.java
b/api/src/main/java/org/apache/iceberg/variants/VariantUtil.java
index 3c88e38157..b6e527bff4 100644
--- a/api/src/main/java/org/apache/iceberg/variants/VariantUtil.java
+++ b/api/src/main/java/org/apache/iceberg/variants/VariantUtil.java
@@ -21,7 +21,7 @@ package org.apache.iceberg.variants;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.charset.StandardCharsets;
-import java.util.function.Function;
+import java.util.function.IntFunction;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.util.ByteBuffers;
@@ -99,19 +99,33 @@ class VariantUtil {
}
}
- static <T extends Comparable<T>> int find(int size, T key, Function<Integer,
T> resolve) {
- int low = 0;
- int high = size - 1;
- while (low <= high) {
- int mid = (low + high) >>> 1;
- T value = resolve.apply(mid);
- int cmp = key.compareTo(value);
- if (cmp == 0) {
- return mid;
- } else if (cmp < 0) {
- high = mid - 1;
- } else {
- low = mid + 1;
+ static int find(int size, String key, IntFunction<String> resolve) {
+ // retry supplementary-plane keys in UTF-16 order to find fields written
in the legacy layout
+ int attempts = 1;
+ for (int i = 0; i < key.length(); i += 1) {
+ if (key.charAt(i) >= Character.MIN_SURROGATE) {
+ attempts = 2;
+ break;
+ }
+ }
+
+ for (int attempt = 0; attempt < attempts; attempt += 1) {
+ int low = 0;
+ int high = size - 1;
+ while (low <= high) {
+ int mid = (low + high) >>> 1;
+ String value = resolve.apply(mid);
+ int cmp =
+ attempt == 0
+ ? VariantMetadata.FIELD_NAME_ORDER.compare(key, value)
+ : key.compareTo(value);
+ if (cmp == 0) {
+ return mid;
+ } else if (cmp < 0) {
+ high = mid - 1;
+ } else {
+ low = mid + 1;
+ }
}
}
diff --git a/api/src/test/java/org/apache/iceberg/variants/VariantTestUtil.java
b/api/src/test/java/org/apache/iceberg/variants/VariantTestUtil.java
index 8a1cd515df..ac1bb2b247 100644
--- a/api/src/test/java/org/apache/iceberg/variants/VariantTestUtil.java
+++ b/api/src/test/java/org/apache/iceberg/variants/VariantTestUtil.java
@@ -163,7 +163,10 @@ public class VariantTestUtil {
}
int numElements = fieldNames.size();
- Stream<String> names = sortNames ? fieldNames.stream().sorted() :
fieldNames.stream();
+ Stream<String> names =
+ sortNames
+ ? fieldNames.stream().sorted(VariantMetadata.FIELD_NAME_ORDER)
+ : fieldNames.stream();
ByteBuffer[] nameBuffers =
names
.map(str -> ByteBuffer.wrap(str.getBytes(StandardCharsets.UTF_8)))
@@ -243,7 +246,10 @@ public class VariantTestUtil {
// write field IDs, values, and offsets
int nextOffset = 0;
int index = 0;
- List<String> sortedFieldNames =
data.keySet().stream().sorted().collect(Collectors.toList());
+ List<String> sortedFieldNames =
+ data.keySet().stream()
+ .sorted(VariantMetadata.FIELD_NAME_ORDER)
+ .collect(Collectors.toList());
for (String fieldName : sortedFieldNames) {
int id = metadata.id(fieldName);
ByteBuffers.writeLittleEndianUnsigned(
diff --git a/core/src/main/java/org/apache/iceberg/variants/ShreddedObject.java
b/core/src/main/java/org/apache/iceberg/variants/ShreddedObject.java
index 7119531aee..7ab88423b0 100644
--- a/core/src/main/java/org/apache/iceberg/variants/ShreddedObject.java
+++ b/core/src/main/java/org/apache/iceberg/variants/ShreddedObject.java
@@ -61,7 +61,8 @@ public class ShreddedObject implements VariantObject,
Serializable {
}
private Set<String> nameSet() {
- Set<String> names = Sets.newTreeSet(shreddedFields.keySet());
+ Set<String> names = Sets.newTreeSet(VariantMetadata.FIELD_NAME_ORDER);
+ names.addAll(shreddedFields.keySet());
if (unshredded != null) {
Iterables.addAll(names, unshredded.fieldNames());
@@ -157,7 +158,7 @@ public class ShreddedObject implements VariantObject,
Serializable {
Set<String> removedFields) {
this.fieldIdSize = VariantUtil.sizeOf(metadata.dictionarySize());
- Map<String, Entry> sorted = Maps.newTreeMap();
+ Map<String, Entry> sorted =
Maps.newTreeMap(VariantMetadata.FIELD_NAME_ORDER);
int totalDataSize = 0;
for (Map.Entry<String, VariantValue> field : shredded.entrySet()) {
diff --git a/core/src/main/java/org/apache/iceberg/variants/Variants.java
b/core/src/main/java/org/apache/iceberg/variants/Variants.java
index 95da533bf3..a7ef465b74 100644
--- a/core/src/main/java/org/apache/iceberg/variants/Variants.java
+++ b/core/src/main/java/org/apache/iceberg/variants/Variants.java
@@ -57,7 +57,7 @@ public class Variants {
for (String name : fieldNames) {
nameBuffers[pos] =
ByteBuffer.wrap(name.getBytes(StandardCharsets.UTF_8));
dataSize += nameBuffers[pos].remaining();
- if (last != null && last.compareTo(name) >= 0) {
+ if (last != null && VariantMetadata.FIELD_NAME_ORDER.compare(last, name)
>= 0) {
sorted = false;
}
diff --git
a/core/src/test/java/org/apache/iceberg/variants/TestShreddedObject.java
b/core/src/test/java/org/apache/iceberg/variants/TestShreddedObject.java
index b5294dc8a5..d0b29dd934 100644
--- a/core/src/test/java/org/apache/iceberg/variants/TestShreddedObject.java
+++ b/core/src/test/java/org/apache/iceberg/variants/TestShreddedObject.java
@@ -65,6 +65,27 @@ public class TestShreddedObject {
assertThat(object.get("c").asPrimitive().get()).isEqualTo(new
BigDecimal("12.21"));
}
+ @Test
+ public void testShreddedFieldOrderFollowsUtf8ByteOrder() {
+ // U+FFFF is 3 UTF-8 bytes (EF BF BF), U+10000 is 4 (F0 90 80 80); they
order oppositely in
+ // UTF-16
+ String threeByteName = new String(Character.toChars(0xFFFF));
+ String fourByteName = new String(Character.toChars(0x10000));
+ VariantMetadata metadata = Variants.metadata(threeByteName, fourByteName);
+ Map<String, VariantValue> fields =
+ ImmutableMap.of(threeByteName, Variants.of(1), fourByteName,
Variants.of(2));
+ ShreddedObject object = createShreddedObject(metadata, fields);
+
+ VariantValue value = roundTripMinimalBuffer(object, metadata);
+
+ assertThat(value).isInstanceOf(SerializedObject.class);
+ SerializedObject actual = (SerializedObject) value;
+ // the on-disk field order is UTF-8 byte order, not the JVM's UTF-16
String order
+ assertThat(actual.fieldNames()).containsExactly(threeByteName,
fourByteName);
+ assertThat(actual.get(threeByteName).asPrimitive().get()).isEqualTo(1);
+ assertThat(actual.get(fourByteName).asPrimitive().get()).isEqualTo(2);
+ }
+
@Test
public void testByteBufferConversion() {
Map<String, VariantValue> pathNormalizedFields =
diff --git
a/core/src/test/java/org/apache/iceberg/variants/TestVariantMetadataFieldOrdering.java
b/core/src/test/java/org/apache/iceberg/variants/TestVariantMetadataFieldOrdering.java
new file mode 100644
index 0000000000..ad3d72ad09
--- /dev/null
+++
b/core/src/test/java/org/apache/iceberg/variants/TestVariantMetadataFieldOrdering.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.variants;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.Test;
+
+public class TestVariantMetadataFieldOrdering {
+
+ // U+FFFF encodes to 3 UTF-8 bytes (EF BF BF), U+10000 to 4 (F0 90 80 80)
+ private static final String NAME_3_BYTE = new
String(Character.toChars(0xFFFF));
+ private static final String NAME_4_BYTE = new
String(Character.toChars(0x10000));
+
+ @Test
+ public void utf8OrderedDictionaryIsSortedAndOrdered() {
+ SerializedMetadata metadata =
+ (SerializedMetadata) Variants.metadata(ImmutableList.of(NAME_3_BYTE,
NAME_4_BYTE));
+
+ assertThat(metadata.isSorted()).isTrue();
+ assertThat(metadata.get(0)).isEqualTo(NAME_3_BYTE);
+ assertThat(metadata.get(1)).isEqualTo(NAME_4_BYTE);
+ assertThat(metadata.id(NAME_3_BYTE)).isEqualTo(0);
+ assertThat(metadata.id(NAME_4_BYTE)).isEqualTo(1);
+ }
+
+ @Test
+ public void utf16OrderedDictionaryIsNotFlaggedSorted() {
+ // UTF-16 order differs from UTF-8: high surrogate D800 sorts before FFFF
+ List<String> utf16Ordered =
+ ImmutableList.of(NAME_3_BYTE,
NAME_4_BYTE).stream().sorted().collect(Collectors.toList());
+ assertThat(utf16Ordered).containsExactly(NAME_4_BYTE, NAME_3_BYTE);
+
+ SerializedMetadata metadata = (SerializedMetadata)
Variants.metadata(utf16Ordered);
+
+ assertThat(metadata.isSorted()).isFalse();
+ // lookups still succeed via the unsorted (linear) path, at their input
positions
+ assertThat(metadata.id(NAME_4_BYTE)).isEqualTo(0);
+ assertThat(metadata.id(NAME_3_BYTE)).isEqualTo(1);
+ }
+
+ @Test
+ public void utf16OrderedDictionaryFlaggedSortedIsFoundViaFallback() {
+ ByteBuffer buffer =
+ VariantTestUtil.createMetadata(ImmutableList.of(NAME_4_BYTE,
NAME_3_BYTE), false);
+ buffer.put(0, (byte) (buffer.get(0) | 0b10000));
+
+ SerializedMetadata metadata = SerializedMetadata.from(buffer);
+
+ assertThat(metadata.isSorted()).isTrue();
+ assertThat(metadata.id(NAME_3_BYTE)).isEqualTo(1);
+ assertThat(metadata.id(NAME_4_BYTE)).isEqualTo(0);
+ }
+}
diff --git
a/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/RecordConverter.java
b/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/RecordConverter.java
index 41e70d6755..e42ed6321f 100644
---
a/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/RecordConverter.java
+++
b/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/RecordConverter.java
@@ -581,7 +581,9 @@ class RecordConverter {
}
List<String> sortedFieldNames =
-
collectFieldNames(value).stream().sorted().collect(Collectors.toList());
+ collectFieldNames(value).stream()
+ .sorted(VariantMetadata.FIELD_NAME_ORDER)
+ .collect(Collectors.toList());
VariantMetadata metadata = Variants.metadata(sortedFieldNames);
return Variant.of(metadata, objectToVariantValue(value, metadata, null));
}
diff --git
a/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestRecordConverter.java
b/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestRecordConverter.java
index 4fec5d914a..8a7272d455 100644
---
a/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestRecordConverter.java
+++
b/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestRecordConverter.java
@@ -1417,6 +1417,23 @@ public class TestRecordConverter {
assertThat(variantConverter().convertVariantValue(original)).isSameAs(original);
}
+ @Test
+ public void testConvertVariantValueOrdersFieldNamesByUtf8() {
+ // U+FFFF is 3 UTF-8 bytes (EF BF BF), U+10000 is 4 (F0 90 80 80); UTF-16
orders them oppositely
+ String threeByteName = new String(Character.toChars(0xFFFF));
+ String fourByteName = new String(Character.toChars(0x10000));
+ Map<String, Object> input = Maps.newLinkedHashMap();
+ input.put(fourByteName, 2);
+ input.put(threeByteName, 1);
+
+ Variant variant = variantConverter().convertVariantValue(input);
+
+ assertThat(variant.metadata().get(0)).isEqualTo(threeByteName);
+ assertThat(variant.metadata().get(1)).isEqualTo(fourByteName);
+
assertThat(variant.value().asObject().get(threeByteName).asPrimitive().get()).isEqualTo(1);
+
assertThat(variant.value().asObject().get(fourByteName).asPrimitive().get()).isEqualTo(2);
+ }
+
@Test
public void testConvertVariantValueFromPrimitiveString() {
Variant variant = variantConverter().convertVariantValue("hello");
diff --git
a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java
b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java
index 346931b375..c45c0a507a 100644
--- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java
+++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java
@@ -413,7 +413,7 @@ class ParquetMetrics {
new FieldMetrics<>(fieldId, metadataCounts.valueCount(),
metadataCounts.nullCount()));
}
- Set<String> fieldNames = Sets.newTreeSet();
+ Set<String> fieldNames =
Sets.newTreeSet(VariantMetadata.FIELD_NAME_ORDER);
for (ParquetVariantUtil.VariantMetrics result : results.subList(1,
results.size())) {
if (result.lowerBound() != null || result.upperBound() != null) {
fieldNames.add(result.fieldName());
diff --git
a/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java
b/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java
index 3e0c088192..e8fb8f1452 100644
--- a/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java
+++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java
@@ -710,6 +710,38 @@ public class TestVariantMetrics {
.isEqualTo(Map.of(1, Types.LongType.get(), 2,
Types.VariantType.get()));
}
+ @Test
+ public void testShreddedObjectBoundFieldsOrderedByUtf8() throws IOException {
+ // U+FFFF is 3 UTF-8 bytes (EF BF BF), U+10000 is 4 (F0 90 80 80); UTF-16
orders them oppositely
+ String threeByteName = new String(Character.toChars(0xFFFF));
+ String fourByteName = new String(Character.toChars(0x10000));
+ VariantMetadata objectMetadata = Variants.metadata(threeByteName,
fourByteName);
+
+ ShreddedObject object0 = Variants.object(objectMetadata);
+ object0.put(threeByteName, Variants.of(1));
+ object0.put(fourByteName, Variants.of(2));
+ ShreddedObject object1 = Variants.object(objectMetadata);
+ object1.put(threeByteName, Variants.of(3));
+ object1.put(fourByteName, Variants.of(4));
+
+ Metrics metrics =
+ writeParquet(
+ (id, name) -> ParquetVariantUtil.toParquetSchema(object0),
+ Variant.of(objectMetadata, object0),
+ Variant.of(objectMetadata, object1));
+
+ String threePath = "$['" + threeByteName + "']";
+ String fourPath = "$['" + fourByteName + "']";
+ Variant lowerBound = Variant.from(metrics.lowerBounds().get(2));
+ assertThat(lowerBound.metadata().get(0)).isEqualTo(threePath);
+ assertThat(lowerBound.metadata().get(1)).isEqualTo(fourPath);
+
assertThat(lowerBound.value().asObject().fieldNames()).containsExactly(threePath,
fourPath);
+ Variant upperBound = Variant.from(metrics.upperBounds().get(2));
+ assertThat(upperBound.metadata().get(0)).isEqualTo(threePath);
+ assertThat(upperBound.metadata().get(1)).isEqualTo(fourPath);
+
assertThat(upperBound.value().asObject().fieldNames()).containsExactly(threePath,
fourPath);
+ }
+
@Test
public void testPartiallyShreddedObject() throws IOException {
VariantValue date = Variants.ofIsoDate("2025-03-17");
