This is an automated email from the ASF dual-hosted git repository. XiaoHongbo-Hope pushed a commit to branch release-2.1 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit a9a3d0f7eb33af20605ff159e02182c27dfbb875 Author: Eunbin Son <[email protected]> AuthorDate: Fri Sep 11 16:05:27 2026 +0900 [codegen] Fix map equaliser for keys without value-based Java equality (#9738) --- .../apache/paimon/codegen/ScalarOperatorGens.scala | 15 +- .../paimon/codegen/EqualiserCodeGeneratorTest.java | 175 +++++++++++++++++++++ 2 files changed, 189 insertions(+), 1 deletion(-) diff --git a/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala b/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala index d2cf7362ba..fa46e3d02a 100644 --- a/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala +++ b/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala @@ -337,7 +337,7 @@ object ScalarOperatorGens { val mapDataUtil = className[InternalMapSerializer] val stmt = - if (containsFloatingPoint(keyType)) { + if (requiresElementWiseKeyMatch(keyType)) { val leftKeyArrayTerm = newName("leftKeyArray") val rightKeyArrayTerm = newName("rightKeyArray") val leftValueArrayTerm = newName("leftValueArray") @@ -472,5 +472,18 @@ object ScalarOperatorGens { case _ => false } + /** + * Whether map keys of the given type must be matched pairwise with the generated key equality + * instead of being looked up through a [[java.util.Map]]. + * + * A [[java.util.Map]] lookup is only correct when the internal representation of the key has + * value-based `equals`/`hashCode`, which holds for numeric, character string, decimal and + * temporal keys. Binary keys are `byte[]` and compare by identity; composite keys may be columnar + * views without `hashCode` support or mix generic and binary representations across the two maps; + * floating-point keys need `Float.compare`/`Double.compare` semantics. + */ + private def requiresElementWiseKeyMatch(keyType: DataType): Boolean = + containsFloatingPoint(keyType) || isBinaryString(keyType) || !isComparable(keyType) + // ---------------------------------------------------------------------------------------------- } diff --git a/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java b/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java index 3d75cdcf41..ab4ad74af8 100644 --- a/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java +++ b/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java @@ -50,6 +50,7 @@ import org.junit.jupiter.params.provider.EnumSource; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; import java.util.concurrent.ThreadLocalRandom; @@ -420,6 +421,180 @@ public class EqualiserCodeGeneratorTest { .isEqualTo(equal); } + @Test + public void testBinaryKeyMapEqualiser() { + DataType mapType = DataTypes.MAP(DataTypes.BYTES(), DataTypes.INT()); + RecordEqualiser equaliser = + new EqualiserCodeGenerator(new DataType[] {mapType}) + .generateRecordEqualiser("binaryKeyMapFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + + // byte[] keys with equal content but distinct identity must compare equal + assertMapEqualiser( + equaliser, + mapType, + new GenericMap(singletonMap("k1".getBytes(), 1)), + new GenericMap(singletonMap("k1".getBytes(), 1)), + true); + assertMapEqualiser( + equaliser, + mapType, + new GenericMap(singletonMap("k1".getBytes(), 1)), + new GenericMap(singletonMap("k2".getBytes(), 1)), + false); + assertMapEqualiser( + equaliser, + mapType, + new GenericMap(singletonMap("k1".getBytes(), 1)), + new GenericMap(singletonMap("k1".getBytes(), 2)), + false); + + // multiple keys: same insertion order on both sides so that the serialized + // BinaryRow bytes are identical for the BinaryRow/BinaryRow fast path + Map<Object, Object> leftMap = new LinkedHashMap<>(); + leftMap.put("k1".getBytes(), 1); + leftMap.put("k2".getBytes(), 2); + Map<Object, Object> rightMap = new LinkedHashMap<>(); + rightMap.put("k1".getBytes(), 1); + rightMap.put("k2".getBytes(), 2); + assertMapEqualiser( + equaliser, mapType, new GenericMap(leftMap), new GenericMap(rightMap), true); + + // entry order must not matter for the element-wise comparison + Map<Object, Object> reversedMap = new LinkedHashMap<>(); + reversedMap.put("k2".getBytes(), 2); + reversedMap.put("k1".getBytes(), 1); + assertThat( + equaliser.equals( + GenericRow.of(new GenericMap(leftMap)), + GenericRow.of(new GenericMap(reversedMap)))) + .isTrue(); + + Map<Object, Object> differentValueMap = new LinkedHashMap<>(); + differentValueMap.put("k1".getBytes(), 1); + differentValueMap.put("k2".getBytes(), 3); + assertMapEqualiser( + equaliser, + mapType, + new GenericMap(leftMap), + new GenericMap(differentValueMap), + false); + } + + @Test + public void testBinaryElementMultisetEqualiser() { + DataType multisetType = DataTypes.MULTISET(DataTypes.BYTES()); + RecordEqualiser equaliser = + new EqualiserCodeGenerator(new DataType[] {multisetType}) + .generateRecordEqualiser("binaryMultisetFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + + assertMapEqualiser( + equaliser, + multisetType, + new GenericMap(singletonMap("e1".getBytes(), 2)), + new GenericMap(singletonMap("e1".getBytes(), 2)), + true); + assertMapEqualiser( + equaliser, + multisetType, + new GenericMap(singletonMap("e1".getBytes(), 2)), + new GenericMap(singletonMap("e2".getBytes(), 2)), + false); + assertMapEqualiser( + equaliser, + multisetType, + new GenericMap(singletonMap("e1".getBytes(), 2)), + new GenericMap(singletonMap("e1".getBytes(), 3)), + false); + } + + @Test + public void testRowKeyMapEqualiserAcrossRowRepresentations() { + DataType mapType = DataTypes.MAP(DataTypes.ROW(DataTypes.INT()), DataTypes.INT()); + RecordEqualiser equaliser = + new EqualiserCodeGenerator(new DataType[] {mapType}) + .generateRecordEqualiser("rowKeyMapFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + + // a binary side yields NestedRow keys while the generic side yields GenericRow keys + assertMapEqualiser( + equaliser, + mapType, + new GenericMap(singletonMap(GenericRow.of(1), 10)), + new GenericMap(singletonMap(GenericRow.of(1), 10)), + true); + assertMapEqualiser( + equaliser, + mapType, + new GenericMap(singletonMap(GenericRow.of(1), 10)), + new GenericMap(singletonMap(GenericRow.of(2), 10)), + false); + assertMapEqualiser( + equaliser, + mapType, + new GenericMap(singletonMap(GenericRow.of(1), 10)), + new GenericMap(singletonMap(GenericRow.of(1), 11)), + false); + } + + @Test + public void testIntKeyMapEqualiserIgnoresEntryOrder() { + DataType mapType = DataTypes.MAP(DataTypes.INT(), DataTypes.INT()); + RecordEqualiser equaliser = + new EqualiserCodeGenerator(new DataType[] {mapType}) + .generateRecordEqualiser("intKeyMapFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + + Map<Object, Object> leftMap = new LinkedHashMap<>(); + leftMap.put(1, 10); + leftMap.put(2, 20); + Map<Object, Object> reversedMap = new LinkedHashMap<>(); + reversedMap.put(2, 20); + reversedMap.put(1, 10); + assertThat( + equaliser.equals( + GenericRow.of(new GenericMap(leftMap)), + GenericRow.of(new GenericMap(reversedMap)))) + .isTrue(); + assertMapEqualiser( + equaliser, mapType, new GenericMap(leftMap), new GenericMap(leftMap), true); + + Map<Object, Object> differentValueMap = new LinkedHashMap<>(); + differentValueMap.put(1, 10); + differentValueMap.put(2, 21); + assertMapEqualiser( + equaliser, + mapType, + new GenericMap(leftMap), + new GenericMap(differentValueMap), + false); + } + + private static Map<Object, Object> singletonMap(Object key, Object value) { + Map<Object, Object> map = new HashMap<>(); + map.put(key, value); + return map; + } + + private static void assertMapEqualiser( + RecordEqualiser equaliser, + DataType mapType, + GenericMap left, + GenericMap right, + boolean equal) { + Serializer<?> serializer = InternalSerializers.create(mapType); + Function<GenericMap, BinaryRow> toBinaryRow = + value -> { + BinaryRow row = new BinaryRow(1); + BinaryRowWriter writer = new BinaryRowWriter(row); + BinaryWriter.write(writer, 0, value, mapType, serializer); + writer.complete(); + return row; + }; + assertBoolean(equaliser, toBinaryRow, left, right, equal); + } + @RepeatedTest(100) public void testManyFields() { int size = 499;
