This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 7dac195327 [core] Handle null literals in IN and ARRAYS_OVERLAP
predicates for file-indexed columns (#10718)
7dac195327 is described below
commit 7dac19532748cc2a634107d47030e77712f66423
Author: cxzl25 <[email protected]>
AuthorDate: Fri Oct 23 09:35:21 2026 +0800
[core] Handle null literals in IN and ARRAYS_OVERLAP predicates for
file-indexed columns (#10718)
---
.../paimon/fileindex/FileIndexPredicate.java | 25 ++++++++++++++++++----
.../paimon/spark/sql/PaimonPushDownTestBase.scala | 19 ++++++++++++++++
2 files changed, 40 insertions(+), 4 deletions(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexPredicate.java
b/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexPredicate.java
index 57ea87b003..b664cb4758 100644
---
a/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexPredicate.java
+++
b/paimon-common/src/main/java/org/apache/paimon/fileindex/FileIndexPredicate.java
@@ -23,8 +23,11 @@ import org.apache.paimon.fs.ByteArraySeekableStream;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.SeekableInputStream;
+import org.apache.paimon.predicate.ArraysOverlap;
import org.apache.paimon.predicate.CompoundPredicate;
import org.apache.paimon.predicate.FieldRef;
+import org.apache.paimon.predicate.In;
+import org.apache.paimon.predicate.LeafFunction;
import org.apache.paimon.predicate.LeafPredicate;
import org.apache.paimon.predicate.Or;
import org.apache.paimon.predicate.Predicate;
@@ -46,10 +49,13 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Optional;
import java.util.Set;
+import static java.util.stream.Collectors.toList;
import static org.apache.paimon.fileindex.FileIndexResult.REMAIN;
+import static org.apache.paimon.fileindex.FileIndexResult.SKIP;
/** Utils to check secondary index (e.g. bloom filter) predicate. */
public class FileIndexPredicate implements Closeable {
@@ -189,14 +195,25 @@ public class FileIndexPredicate implements Closeable {
return REMAIN;
}
+ LeafFunction function = predicate.function();
+ List<Object> literals = predicate.literals();
+ if (literals.stream().anyMatch(Objects::isNull)) {
+ // Index readers do not expect a null literal. A null literal
never matches a
+ // row: IN and ARRAYS_OVERLAP ignore it, and every other
function is false on it.
+ if (function instanceof In || function instanceof
ArraysOverlap) {
+ literals =
literals.stream().filter(Objects::nonNull).collect(toList());
+ if (literals.isEmpty()) {
+ return SKIP;
+ }
+ } else {
+ return SKIP;
+ }
+ }
FileIndexResult compoundResult = REMAIN;
FieldRef fieldRef = fieldRefOptional.get();
for (FileIndexReader fileIndexReader :
columnIndexReaders.get(fieldRef.name())) {
compoundResult =
- compoundResult.and(
- predicate
- .function()
- .visit(fileIndexReader, fieldRef,
predicate.literals()));
+ compoundResult.and(function.visit(fileIndexReader,
fieldRef, literals));
if (!compoundResult.remain()) {
return compoundResult;
diff --git
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonPushDownTestBase.scala
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonPushDownTestBase.scala
index 71fce5b4da..9ff06e54ca 100644
---
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonPushDownTestBase.scala
+++
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/PaimonPushDownTestBase.scala
@@ -611,6 +611,25 @@ abstract class PaimonPushDownTestBase extends
PaimonSparkTestBase with AdaptiveS
}
}
+ test("Paimon pushDown: IN with a NULL literal on a file-indexed column") {
+ Seq("range-bitmap", "bsi", "bitmap").foreach {
+ indexType =>
+ withTable("T") {
+ sql(s"""
+ |CREATE TABLE T (id INT, v STRING)
+ |TBLPROPERTIES ('file-index.$indexType.columns' = 'id',
'file-index.in-manifest-threshold' = '1 kb')
+ |""".stripMargin)
+ sql("INSERT INTO T VALUES (1, 'a'), (NULL, 'b'), (3, 'c')")
+
+ checkAnswer(sql("SELECT v FROM T WHERE id IN (1, NULL)"), Row("a")
:: Nil)
+ checkAnswer(sql("SELECT v FROM T WHERE id NOT IN (1, NULL)"), Nil)
+ checkAnswer(
+ sql("SELECT v FROM T WHERE id IN (3, NULL) OR id IS NULL ORDER BY
v"),
+ Row("b") :: Row("c") :: Nil)
+ }
+ }
+ }
+
test("Paimon pushDown: TopN for append-only tables") {
assume(gteqSpark3_3)
spark.sql("""
