This is an automated email from the ASF dual-hosted git repository. XiaoHongbo-Hope pushed a commit to branch release-2.1 in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 79b6f948ea8ebfb0853307e2e7895004ee183763 Author: YangJie <[email protected]> AuthorDate: Fri Sep 11 03:13:29 2026 -0400 [common] Skip null literals in btree global index IN predicates (#9624) --- .../paimon/globalindex/btree/BTreeIndexReader.java | 5 ++++ .../globalindex/btree/AbstractIndexReaderTest.java | 24 +++++++++++++++++++ .../procedure/CreateGlobalIndexProcedureTest.scala | 28 ++++++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexReader.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexReader.java index 6203d6d4a0..b0c272321c 100644 --- a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexReader.java +++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeIndexReader.java @@ -334,6 +334,11 @@ public class BTreeIndexReader implements Closeable { () -> { RoaringNavigableMap64 result = new RoaringNavigableMap64(); for (Object literal : literals) { + // SQL IN treats NULL as never matching; skip it instead of + // failing to serialize a null key. + if (literal == null) { + continue; + } result.or(rangeQuery(literal, literal, true, true)); } return result; diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/AbstractIndexReaderTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/AbstractIndexReaderTest.java index d35490e1fd..fe7ad8b40e 100644 --- a/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/AbstractIndexReaderTest.java +++ b/paimon-common/src/test/java/org/apache/paimon/globalindex/btree/AbstractIndexReaderTest.java @@ -276,6 +276,30 @@ public abstract class AbstractIndexReaderTest { } } + @TestTemplate + public void testInPredicateWithNullLiteral() throws Exception { + FieldRef ref = new FieldRef(1, "testField", dataType); + + try (GlobalIndexReader reader = prepareDataAndCreateReader()) { + // Mixed null literal: SQL IN never matches NULL, so the null is ignored + // and the remaining literals drive the result (previously NPE'd). + List<Object> literals = + new ArrayList<>( + data.get(0).getKey() == null + ? Collections.emptyList() + : Collections.singletonList(data.get(0).getKey())); + literals.add(null); + GlobalIndexResult result = reader.visitIn(ref, literals).join().get(); + List<Object> finalLiterals = literals; + assertResult(result, filter(finalLiterals::contains)); + + // All-null list matches nothing. + GlobalIndexResult allNull = + reader.visitIn(ref, Collections.singletonList(null)).join().get(); + assertThat(allNull.results().isEmpty()).isTrue(); + } + } + @TestTemplate public void testStartsWith() throws Exception { if (!dataType.is(DataTypeFamily.CHARACTER_STRING)) { diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedureTest.scala index 90c1eda5cb..d051bc90dc 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedureTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/CreateGlobalIndexProcedureTest.scala @@ -448,6 +448,34 @@ class CreateGlobalIndexProcedureTest extends PaimonSparkTestBase with StreamTest } } + test("btree global index IN predicate with more than 20 literals including NULL") { + withTable("T") { + spark.sql(""" + |CREATE TABLE T (id INT, idx INT) + |TBLPROPERTIES ( + | 'bucket' = '-1', + | 'global-index.enabled' = 'true', + | 'row-tracking.enabled' = 'true', + | 'data-evolution.enabled' = 'true', + | 'btree-index.records-per-range' = '2') + |""".stripMargin) + + spark.sql(s"INSERT INTO T VALUES ${(0 until 100).map(i => s"($i, $i)").mkString(",")}") + createBTreeIndex("T", "idx") + + // 21 distinct non-null literals plus NULL is 22 (> 20), so PredicateBuilder keeps a real In + // leaf that reaches BTreeIndexReader.visitIn with a null (previously an NPE). Pin + // inSetConversionThreshold high so the pushed predicate stays a plain Catalyst In whose + // translation carries the null. + val inList = ((0 to 20).map(_.toString) :+ "NULL").mkString(", ") + withSQLConf("spark.sql.optimizer.inSetConversionThreshold" -> "100") { + checkAnswer( + sql(s"SELECT id FROM T WHERE idx IN ($inList) ORDER BY id"), + (0 to 20).map(Row(_))) + } + } + } + private def createBTreeIndex(tableName: String, column: String): Unit = { spark .sql(
