This is an automated email from the ASF dual-hosted git repository. XiaoHongbo-Hope pushed a commit to branch release-2.1 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit f080135748465f1390d8046b7367dec168d56cd9 Author: Eunbin Son <[email protected]> AuthorDate: Thu Sep 17 15:45:27 2026 +0900 [codegen] Fix equaliser treating null array elements and map values as unequal (#9860) --- .../apache/paimon/codegen/ScalarOperatorGens.scala | 17 ++++- .../paimon/codegen/EqualiserCodeGeneratorTest.java | 78 ++++++++++++++++++++++ 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala b/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala index fa46e3d02a..2397b0e060 100644 --- a/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala +++ b/paimon-codegen/src/main/scala/org/apache/paimon/codegen/ScalarOperatorGens.scala @@ -272,7 +272,7 @@ object ScalarOperatorGens { | } | | ${elementEqualsExpr.code} - | if (!${elementEqualsExpr.resultTerm}) { + | if (!${nullSafeEquals(leftElementNullTerm, rightElementNullTerm, elementEqualsExpr.resultTerm)}) { | $resultTerm = false; | break; | } @@ -396,7 +396,7 @@ object ScalarOperatorGens { | } | | ${valueEqualsExpr.code} - | if (${valueEqualsExpr.resultTerm}) { + | if (${nullSafeEquals(leftValueNullTerm, rightValueNullTerm, valueEqualsExpr.resultTerm)}) { | $matchedTerm[$rightIndexTerm] = true; | $foundTerm = true; | } @@ -432,7 +432,7 @@ object ScalarOperatorGens { | boolean $rightValueNullTerm = ($rightValueTerm == null); | | ${valueEqualsExpr.code} - | if (!${valueEqualsExpr.resultTerm}) { + | if (!${nullSafeEquals(leftValueNullTerm, rightValueNullTerm, valueEqualsExpr.resultTerm)}) { | $resultTerm = false; | break; | } @@ -464,6 +464,17 @@ object ScalarOperatorGens { } } + /** + * Null-safe equality on a nested element or value: two nulls are equal, a single null is not, + * otherwise the generated equality decides. `generateEquals` leaves its result flag `false` when + * either operand is null, so reading that flag alone treats two null elements as unequal. + */ + private def nullSafeEquals( + leftNullTerm: String, + rightNullTerm: String, + equalsTerm: String): String = + s"(($leftNullTerm && $rightNullTerm) || (!$leftNullTerm && !$rightNullTerm && $equalsTerm))" + private def containsFloatingPoint(t: DataType): Boolean = t.getTypeRoot match { case FLOAT | DOUBLE => true case DataTypeRoot.ARRAY | DataTypeRoot.MAP | DataTypeRoot.MULTISET | DataTypeRoot.ROW | diff --git a/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java b/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java index ab4ad74af8..fc0716903c 100644 --- a/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java +++ b/paimon-codegen/src/test/java/org/apache/paimon/codegen/EqualiserCodeGeneratorTest.java @@ -571,6 +571,84 @@ public class EqualiserCodeGeneratorTest { false); } + @Test + public void testArrayEqualiserTreatsNullElementsAsEqual() { + DataType intArrayType = DataTypes.ARRAY(DataTypes.INT()); + RecordEqualiser intArrayEqualiser = + new EqualiserCodeGenerator(new DataType[] {intArrayType}) + .generateRecordEqualiser("nullElementIntArrayFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + assertFloatingPointEqualiser( + intArrayEqualiser, + intArrayType, + new GenericArray(new Object[] {1, null}), + new GenericArray(new Object[] {1, null}), + true); + assertFloatingPointEqualiser( + intArrayEqualiser, + intArrayType, + new GenericArray(new Object[] {1, null}), + new GenericArray(new Object[] {1, 2}), + false); + + DataType doubleArrayType = DataTypes.ARRAY(DataTypes.DOUBLE()); + RecordEqualiser doubleArrayEqualiser = + new EqualiserCodeGenerator(new DataType[] {doubleArrayType}) + .generateRecordEqualiser("nullElementDoubleArrayFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + assertFloatingPointEqualiser( + doubleArrayEqualiser, + doubleArrayType, + new GenericArray(new Object[] {1.0d, null}), + new GenericArray(new Object[] {1.0d, null}), + true); + + DataType rowArrayType = DataTypes.ARRAY(DataTypes.ROW(DataTypes.INT())); + RecordEqualiser rowArrayEqualiser = + new EqualiserCodeGenerator(new DataType[] {rowArrayType}) + .generateRecordEqualiser("nullElementRowArrayFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + assertFloatingPointEqualiser( + rowArrayEqualiser, + rowArrayType, + new GenericArray(new Object[] {null}), + new GenericArray(new Object[] {null}), + true); + } + + @Test + public void testMapEqualiserTreatsNullValuesAsEqual() { + DataType intKeyMapType = DataTypes.MAP(DataTypes.INT(), DataTypes.INT()); + RecordEqualiser intKeyMapEqualiser = + new EqualiserCodeGenerator(new DataType[] {intKeyMapType}) + .generateRecordEqualiser("nullValueIntKeyMapFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + assertMapEqualiser( + intKeyMapEqualiser, + intKeyMapType, + new GenericMap(singletonMap(1, null)), + new GenericMap(singletonMap(1, null)), + true); + assertMapEqualiser( + intKeyMapEqualiser, + intKeyMapType, + new GenericMap(singletonMap(1, null)), + new GenericMap(singletonMap(1, 2)), + false); + + DataType binaryKeyMapType = DataTypes.MAP(DataTypes.BYTES(), DataTypes.INT()); + RecordEqualiser binaryKeyMapEqualiser = + new EqualiserCodeGenerator(new DataType[] {binaryKeyMapType}) + .generateRecordEqualiser("nullValueBinaryKeyMapFieldEquals") + .newInstance(Thread.currentThread().getContextClassLoader()); + assertMapEqualiser( + binaryKeyMapEqualiser, + binaryKeyMapType, + new GenericMap(singletonMap("k".getBytes(), null)), + new GenericMap(singletonMap("k".getBytes(), null)), + true); + } + private static Map<Object, Object> singletonMap(Object key, Object value) { Map<Object, Object> map = new HashMap<>(); map.put(key, value);
