This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new f797982eb6 [spark] convert EqualTo(NaN) to isNaN in V2 filters (#8542)
f797982eb6 is described below
commit f797982eb6d87c293289508f7a349c9320bb92a6
Author: cxzl25 <[email protected]>
AuthorDate: Mon Jul 13 10:35:57 2026 +0800
[spark] convert EqualTo(NaN) to isNaN in V2 filters (#8542)
---
.../paimon/spark/SparkV2FilterConverter.scala | 13 +++++++--
.../spark/sql/SparkV2FilterConverterTestBase.scala | 32 ++++++++++++++++++++++
2 files changed, 43 insertions(+), 2 deletions(-)
diff --git
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkV2FilterConverter.scala
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkV2FilterConverter.scala
index 1493dfc49c..edece9340b 100644
---
a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkV2FilterConverter.scala
+++
b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/paimon/spark/SparkV2FilterConverter.scala
@@ -49,8 +49,11 @@ case class SparkV2FilterConverter(rowType: RowType) extends
Logging {
case EQUAL_TO =>
sparkPredicate match {
case BinaryPredicate(transform, literal) =>
- // TODO deal with isNaN
- builder.equal(transform, literal)
+ if (isNaN(literal)) {
+ builder.isNaN(transform)
+ } else {
+ builder.equal(transform, literal)
+ }
case _ =>
throw new UnsupportedOperationException(s"Convert $sparkPredicate
is unsupported.")
}
@@ -236,4 +239,10 @@ object SparkV2FilterConverter extends Logging {
private val ALWAYS_TRUE = "ALWAYS_TRUE"
private val ALWAYS_FALSE = "ALWAYS_FALSE"
+ private def isNaN(value: Object): Boolean = value match {
+ case f: java.lang.Float => f.isNaN
+ case d: java.lang.Double => d.isNaN
+ case _ => false
+ }
+
}
diff --git
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala
index f7e0bba63f..ea80fe476d 100644
---
a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala
+++
b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/SparkV2FilterConverterTestBase.scala
@@ -217,6 +217,38 @@ abstract class SparkV2FilterConverterTestBase extends
PaimonSparkTestBase {
assert(scanFilesCount(filter) == 1)
}
+ test("V2Filter: EqualTo NaN") {
+ sql("CREATE TABLE nan_tbl (id INT, f FLOAT, d DOUBLE) USING paimon")
+ // Separate files so finite-range stats would incorrectly prune the NaN
file under equal(NaN).
+ sql("INSERT INTO nan_tbl VALUES (1, 1.0, 1.0)")
+ sql("INSERT INTO nan_tbl VALUES (2, CAST('NaN' AS FLOAT), CAST('NaN' AS
DOUBLE))")
+ sql("INSERT INTO nan_tbl VALUES (3, 3.0, 3.0)")
+
+ val nanRowType = loadTable("nan_tbl").rowType()
+ val nanBuilder = new PredicateBuilder(nanRowType)
+ val nanConverter = SparkV2FilterConverter(nanRowType)
+
+ var filter = "f = CAST('NaN' AS FLOAT)"
+ var actual = nanConverter.convert(v2Filter(filter, "nan_tbl")).get
+ assert(actual.equals(nanBuilder.isNaN(1)))
+ checkAnswer(sql(s"SELECT id FROM nan_tbl WHERE $filter"), Seq(Row(2)))
+ assert(scanFilesCount(filter, "nan_tbl") >= 1)
+
+ filter = "d = CAST('NaN' AS DOUBLE)"
+ actual = nanConverter.convert(v2Filter(filter, "nan_tbl")).get
+ assert(actual.equals(nanBuilder.isNaN(2)))
+ checkAnswer(sql(s"SELECT id FROM nan_tbl WHERE $filter"), Seq(Row(2)))
+ assert(scanFilesCount(filter, "nan_tbl") >= 1)
+
+ // Non-NaN equality is unchanged.
+ filter = "f = 1.0"
+ actual = nanConverter.convert(v2Filter(filter, "nan_tbl")).get
+ assert(actual.equals(nanBuilder.equal(1, 1.0f)))
+ checkAnswer(sql(s"SELECT id FROM nan_tbl WHERE $filter"), Seq(Row(1)))
+
+ sql("DROP TABLE IF EXISTS nan_tbl")
+ }
+
test("V2Filter: EqualNullSafe") {
var filter = "int_col <=> 1"
var actual = converter.convert(v2Filter(filter)).get
